From 2b633a107d1b6d55b79d73590a56dab5aa37f3ef Mon Sep 17 00:00:00 2001 From: Ivan Kovnatsky Date: Thu, 2 May 2019 12:48:09 +0300 Subject: [PATCH 001/962] Fix small typo (#367) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e882fe9a06..cd9fb111d0 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ module "my-cluster" { ## Other documentation -- [Autoscaling](docs/autoscaling.md): How to enabled worker node autoscaling. +- [Autoscaling](docs/autoscaling.md): How to enable worker node autoscaling. - [Enable Docker Bridge Network](docs/enable-docker-bridge-network.md): How to enable the docker bridge network when using the EKS-optimized AMI, which disables it by default. ## Release schedule From 3ece32fedfe0e1a6dd60477b29d7797353d8842b Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 7 May 2019 10:59:07 +0200 Subject: [PATCH 002/962] Fixes for Launch Templates (#361) * Fixes for Launch Templates * update changelog * add outputs for 3 LT attributes --- CHANGELOG.md | 1 + README.md | 7 +++++-- local.tf | 2 +- outputs.tf | 15 +++++++++++++++ workers_launch_template.tf | 11 +++++++++-- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6459488cce..164dc95f69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed + - 4 small changes to `aws_launch_template` resource (by @max-rocket-internet) - Add .prettierignore file (by @rothandrew) - Switch to https for the pre-commit repos (by @rothandrew) - Add instructions on how to enable the docker bridge network (by @rothandrew) diff --git a/README.md b/README.md index cd9fb111d0..11b269cad5 100644 --- a/README.md +++ b/README.md @@ -171,12 +171,15 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | -| worker\_iam\_instance\_profile\_arns | default IAM instance profile ARNs for EKS worker group | -| worker\_iam\_instance\_profile\_names | default IAM instance profile names for EKS worker group | +| worker\_iam\_instance\_profile\_arns | default IAM instance profile ARN for EKS worker groups | +| worker\_iam\_instance\_profile\_names | default IAM instance profile name for EKS worker groups | | worker\_iam\_role\_arn | default IAM role ARN for EKS worker groups | | worker\_iam\_role\_name | default IAM role name for EKS worker groups | | worker\_security\_group\_id | Security group ID attached to the EKS workers. | | workers\_asg\_arns | IDs of the autoscaling groups containing workers. | | workers\_asg\_names | Names of the autoscaling groups containing workers. | +| workers\_launch\_template\_arns | ARNs of the worker launch templates. | +| workers\_launch\_template\_ids | IDs of the worker launch templates. | +| workers\_launch\_template\_latest\_versions | Latest versions of the worker launch templates. | diff --git a/local.tf b/local.tf index 8cb8923d15..d09ed7f57b 100644 --- a/local.tf +++ b/local.tf @@ -64,7 +64,7 @@ locals { root_volume_size = "100" # root volume size of workers instances. root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp2', or 'io1' root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". - root_encrypted = false # root volume encryption for workers. + root_encrypted = "" # root volume encryption for workers. kms_key_id = "" # KMS key ID used for encrypted block device. ASG must have access to this key. If not specified, the default KMS key will be used. key_name = "" # The key name that should be used for the instances in the autoscaling group pre_userdata = "" # userdata to pre-append to the default userdata. diff --git a/outputs.tf b/outputs.tf index 1521d5c379..83a0c11543 100644 --- a/outputs.tf +++ b/outputs.tf @@ -64,6 +64,21 @@ output "workers_asg_names" { value = "${concat(aws_autoscaling_group.workers.*.id, aws_autoscaling_group.workers_launch_template.*.id)}" } +output "workers_launch_template_ids" { + description = "IDs of the worker launch templates." + value = "${aws_launch_template.workers_launch_template.*.id}" +} + +output "workers_launch_template_arns" { + description = "ARNs of the worker launch templates." + value = "${aws_launch_template.workers_launch_template.*.arn}" +} + +output "workers_launch_template_latest_versions" { + description = "Latest versions of the worker launch templates." + value = "${aws_launch_template.workers_launch_template.*.latest_version}" +} + output "worker_security_group_id" { description = "Security group ID attached to the EKS workers." value = "${local.worker_security_group_id}" diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 19804a8985..c28ca741bc 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -66,10 +66,15 @@ resource "aws_launch_template" "workers_launch_template" { network_interfaces { associate_public_ip_address = "${lookup(var.worker_groups_launch_template[count.index], "public_ip", local.workers_group_launch_template_defaults["public_ip"])}" delete_on_termination = "${lookup(var.worker_groups_launch_template[count.index], "eni_delete", local.workers_group_launch_template_defaults["eni_delete"])}" - security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",",lookup(var.worker_groups_launch_template[count.index],"additional_security_group_ids", local.workers_group_launch_template_defaults["additional_security_group_ids"])))}"] + + security_groups = [ + "${local.worker_security_group_id}", + "${var.worker_additional_security_group_ids}", + "${compact(split(",",lookup(var.worker_groups_launch_template[count.index],"additional_security_group_ids", local.workers_group_launch_template_defaults["additional_security_group_ids"])))}", + ] } - iam_instance_profile = { + iam_instance_profile { name = "${element(aws_iam_instance_profile.workers_launch_template.*.name, count.index)}" } @@ -106,6 +111,8 @@ resource "aws_launch_template" "workers_launch_template" { delete_on_termination = true } } + + tags = "${var.tags}" } resource "aws_iam_instance_profile" "workers_launch_template" { From b27b58257d78bee660ee6c700f831d1e5aaaed34 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 7 May 2019 10:59:43 +0200 Subject: [PATCH 003/962] Adding doc about spot instances (#362) * Adding doc about spot instances * updating LT worker group details * updating doc --- CHANGELOG.md | 3 +- README.md | 1 + docs/spot-instances.md | 105 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 docs/spot-instances.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 164dc95f69..fc8e17c0fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added -- Added support for custom service linked role for Auto Scaling group (by @voanhduy1512) +- Added doc about spot instances (by @max-rocket-internet) +- Added support for custom service linked role for Auto Scaling group (by @voanhduy1512) - Write your awesome addition here (by @you) ### Changed diff --git a/README.md b/README.md index 11b269cad5..13df9ee052 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ module "my-cluster" { - [Autoscaling](docs/autoscaling.md): How to enable worker node autoscaling. - [Enable Docker Bridge Network](docs/enable-docker-bridge-network.md): How to enable the docker bridge network when using the EKS-optimized AMI, which disables it by default. +- [Spot instances](docs/spot-instances.md): How to use spot instances with this module. ## Release schedule diff --git a/docs/spot-instances.md b/docs/spot-instances.md new file mode 100644 index 0000000000..1cc0be1a4b --- /dev/null +++ b/docs/spot-instances.md @@ -0,0 +1,105 @@ +# Using spot instances + +Spot instances usually cost around 30-70% less than an on-demand instance. So using them for your EKS workloads can save a lot of money but requires some special considerations as they will be terminated with only 2 minutes warning. + +You need to install a daemonset to catch the 2 minute warning before termination. This will ensure the node is gracefully drained before termination. You can install the [k8s-spot-termination-handler](https://github.com/kube-aws/kube-spot-termination-notice-handler) for this. There's a [Helm chart](https://github.com/helm/charts/tree/master/stable/k8s-spot-termination-handler): + +``` +helm install stable/k8s-spot-termination-handler --namespace kube-system +``` + +In the following examples at least 1 worker group that uses on-demand instances is included. This worker group has an added node label that can be used in scheduling. This could be used to schedule any workload but is important for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) as it might be end up unscheduled when spot instances are terminated. You can add this to the values of the [cluster-autoscaler helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler): + +```yaml +nodeSelector: + spot: "false" +``` + +Notes: + +- The `spot_price` is set to the on-demand price so that the spot instances will run as long as they are the cheaper. +- It's best to have a broad range of instance types to ensure there's always some instances to run when prices fluctuate. +- Using an [AWS Spot Fleet](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet-requests.html) is the best option but is not supported by this module yet. +- There is an AWS blog article about this [here](https://aws.amazon.com/blogs/compute/run-your-kubernetes-workloads-on-amazon-ec2-spot-instances-with-amazon-eks/). +- Consider using [k8s-spot-rescheduler](https://github.com/pusher/k8s-spot-rescheduler) to move pods from on-demand to spot instances. + +## Using Launch Configuration + +Example Terraform worker group configuration that use an ASG with launch configuration: + +```hcl +worker_group_count = 3 + +worker_groups = [ + { + name = "on-demand-1" + instance_type = "m4.xlarge" + asg_max_size = 1 + autoscaling_enabled = true + kubelet_extra_args = "--node-labels=spot=false" + suspended_processes = "AZRebalance" + }, + { + name = "spot-1" + spot_price = "0.39" + instance_type = "c4.2xlarge" + asg_max_size = 20 + autoscaling_enabled = true + kubelet_extra_args = "--node-labels=spot=true" + suspended_processes = "AZRebalance" + }, + { + name = "spot-2" + spot_price = "0.40" + instance_type = "m4.2xlarge" + asg_max_size = 20 + autoscaling_enabled = true + kubelet_extra_args = "--node-labels=spot=true" + suspended_processes = "AZRebalance" + } +] +``` + +## Using Launch Templates + +Launch Template support is a recent addition to both AWS and this module. It might not be as tried and tested. + +Example Terraform worker group configuration that use an ASG with a launch template: + +```hcl + +worker_group_count = 1 + +worker_groups = [ + { + name = "on-demand-1" + instance_type = "m4.xlarge" + asg_max_size = 10 + autoscaling_enabled = true + kubelet_extra_args = "--node-labels=spot=false" + suspended_processes = "AZRebalance" + } +] + +worker_group_launch_template_count = 1 + +worker_groups_launch_template = [ + { + name = "spot-1" + instance_type = "m5.xlarge" + override_instance_type = "m4.xlarge" + spot_instance_pools = 2 + on_demand_percentage_above_base_capacity = 0 + spot_max_price = "0.384" + asg_max_size = 10 + autoscaling_enabled = true + kubelet_extra_args = "--node-labels=spot=true" + } +] +``` + +## Important issues + +- https://github.com/terraform-aws-modules/terraform-aws-eks/issues/360 +- https://github.com/terraform-providers/terraform-provider-aws/issues/8475 +- https://github.com/kubernetes/autoscaler/issues/1133 From 1660105970ff8cef7b7a4e68d2140f53989e0004 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 7 May 2019 11:00:49 +0200 Subject: [PATCH 004/962] Disabling ASG process AZRebalance by default (#369) --- CHANGELOG.md | 1 + local.tf | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc8e17c0fc..fafa4c1d06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed + - Set default suspended processes for ASG to `AZRebalance` (by @max-rocket-internet) - 4 small changes to `aws_launch_template` resource (by @max-rocket-internet) - Add .prettierignore file (by @rothandrew) - Switch to https for the pre-commit repos (by @rothandrew) diff --git a/local.tf b/local.tf index d09ed7f57b..3f6149d05c 100644 --- a/local.tf +++ b/local.tf @@ -35,7 +35,7 @@ locals { additional_security_group_ids = "" # A comma delimited list of additional security group ids to include in worker launch config protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. iam_role_id = "${local.default_iam_role_id}" # Use the specified IAM role if set. - suspended_processes = "" # A comma delimited string of processes to to suspend. i.e. AZRebalance,HealthCheck,ReplaceUnhealthy + suspended_processes = "AZRebalance" # A comma delimited string of processes to to suspend. i.e. AZRebalance,HealthCheck,ReplaceUnhealthy target_group_arns = "" # A comma delimited list of ALB target group ARNs to be associated to the ASG enabled_metrics = "" # A comma delimited list of metrics to be collected i.e. GroupMinSize,GroupMaxSize,GroupDesiredCapacity placement_group = "" # The name of the placement group into which to launch the instances, if any. @@ -80,7 +80,7 @@ locals { additional_security_group_ids = "" # A comma delimited list of additional security group ids to include in worker launch config protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. iam_role_id = "${local.default_iam_role_id}" # Use the specified IAM role if set. - suspended_processes = "" # A comma delimited string of processes to to suspend. i.e. AZRebalance,HealthCheck,ReplaceUnhealthy + suspended_processes = "AZRebalance" # A comma delimited string of processes to to suspend. i.e. AZRebalance,HealthCheck,ReplaceUnhealthy target_group_arns = "" # A comma delimited list of ALB target group ARNs to be associated to the ASG enabled_metrics = "" # A comma delimited list of metrics to be collected i.e. GroupMinSize,GroupMaxSize,GroupDesiredCapacity placement_group = "" # The name of the placement group into which to launch the instances, if any. From 60dfeca191cc60f14e107bd1c8a002360b7cf621 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 7 May 2019 11:09:44 +0200 Subject: [PATCH 005/962] Adding 2 new outputs: AMI ID and work user-data (#364) * Adding 2 new outputs: AMI ID and work user-data * adding separate outputs to include both --- CHANGELOG.md | 2 +- README.md | 2 ++ outputs.tf | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fafa4c1d06..ffbfb94ae1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,9 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added +- Added outputs for `workers_user_data` and `workers_default_ami_id` (by @max-rocket-internet) - Added doc about spot instances (by @max-rocket-internet) - Added support for custom service linked role for Auto Scaling group (by @voanhduy1512) -- Write your awesome addition here (by @you) ### Changed diff --git a/README.md b/README.md index 13df9ee052..4a01f89910 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_security\_group\_id | Security group ID attached to the EKS workers. | | workers\_asg\_arns | IDs of the autoscaling groups containing workers. | | workers\_asg\_names | Names of the autoscaling groups containing workers. | +| workers\_default\_ami\_id | ID of the default worker group AMI | +| workers\_user\_data | User data of worker groups | | workers\_launch\_template\_arns | ARNs of the worker launch templates. | | workers\_launch\_template\_ids | IDs of the worker launch templates. | | workers\_launch\_template\_latest\_versions | Latest versions of the worker launch templates. | diff --git a/outputs.tf b/outputs.tf index 83a0c11543..55d20fb94e 100644 --- a/outputs.tf +++ b/outputs.tf @@ -64,6 +64,16 @@ output "workers_asg_names" { value = "${concat(aws_autoscaling_group.workers.*.id, aws_autoscaling_group.workers_launch_template.*.id)}" } +output "workers_user_data" { + description = "User data of worker groups" + value = "${concat(data.template_file.userdata.*.rendered, data.template_file.launch_template_userdata.*.rendered)}" +} + +output "workers_default_ami_id" { + description = "ID of the default worker group AMI" + value = "${data.aws_ami.eks_worker.id}" +} + output "workers_launch_template_ids" { description = "IDs of the worker launch templates." value = "${aws_launch_template.workers_launch_template.*.id}" From 613fb1ca96375d8b2d80bbd65d9c1105acbaf318 Mon Sep 17 00:00:00 2001 From: Alex Snast Date: Tue, 7 May 2019 15:16:32 +0300 Subject: [PATCH 006/962] adding cluster arn to outputs (#372) --- CHANGELOG.md | 3 ++- outputs.tf | 9 ++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffbfb94ae1..b3612a2c25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,10 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added +- Added support for custom service linked role for Auto Scaling group (by @voanhduy1512) +- Add cluster arn to outputs (by @alexsn) - Added outputs for `workers_user_data` and `workers_default_ami_id` (by @max-rocket-internet) - Added doc about spot instances (by @max-rocket-internet) -- Added support for custom service linked role for Auto Scaling group (by @voanhduy1512) ### Changed diff --git a/outputs.tf b/outputs.tf index 55d20fb94e..2bca30937f 100644 --- a/outputs.tf +++ b/outputs.tf @@ -3,11 +3,10 @@ output "cluster_id" { value = "${aws_eks_cluster.this.id}" } -# Though documented, not yet supported -# output "cluster_arn" { -# description = "The Amazon Resource Name (ARN) of the cluster." -# value = "${aws_eks_cluster.this.arn}" -# } +output "cluster_arn" { + description = "The Amazon Resource Name (ARN) of the cluster." + value = "${aws_eks_cluster.this.arn}" +} output "cluster_certificate_authority_data" { description = "Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster." From 959e5330e35702666fd919d962a5847c3391c0e8 Mon Sep 17 00:00:00 2001 From: Touch Ungboriboonpisal Date: Tue, 7 May 2019 05:46:06 -0700 Subject: [PATCH 007/962] Support custom IAM roles for cluster and workers (#338) * allow specifying custom worker IAM instance profiles * allow specifying custom cluster IAM role * add doc * update changelog * use data.aws_iam_instance_profile.name --- CHANGELOG.md | 1 + README.md | 3 +++ aws_auth.tf | 4 ++-- cluster.tf | 5 ++++- data.tf | 15 +++++++++++++++ local.tf | 9 +++++++-- outputs.tf | 8 ++++---- variables.tf | 15 +++++++++++++++ workers.tf | 12 +++++++++--- workers_launch_template.tf | 4 ++-- 10 files changed, 62 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3612a2c25..41af2c1439 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added - Added support for custom service linked role for Auto Scaling group (by @voanhduy1512) +- Added support for custom IAM roles for cluster and workers (by @erks) - Add cluster arn to outputs (by @alexsn) - Added outputs for `workers_user_data` and `workers_default_ami_id` (by @max-rocket-internet) - Added doc about spot instances (by @max-rocket-internet) diff --git a/README.md b/README.md index 4a01f89910..cd3e9e18a8 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | list | `[]` | no | | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | string | `"false"` | no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | string | `"true"` | no | +| cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. | string | `""` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers and provide API access to your current IP/32. | string | `""` | no | | cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.12"` | no | @@ -130,6 +131,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | kubeconfig\_name | Override the default name used for items kubeconfig. | string | `""` | no | | local\_exec\_interpreter | Command to run for local-exec resources. Must be a shell-style interpreter. If you are on Windows Git Bash is a good choice. | list | `[ "/bin/sh", "-c" ]` | no | | manage\_aws\_auth | Whether to apply the aws-auth configmap file. | string | `"true"` | no | +| manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | string | `"true"` | no | +| manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | string | `"true"` | no | | map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/eks_test_fixture/variables.tf for example format. | list | `[]` | no | | map\_accounts\_count | The count of accounts in the map_accounts list. | string | `"0"` | no | | map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/eks_test_fixture/variables.tf for example format. | list | `[]` | no | diff --git a/aws_auth.tf b/aws_auth.tf index abe8d0766f..a86349c52c 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -39,7 +39,7 @@ data "template_file" "launch_template_worker_role_arns" { template = "${file("${path.module}/templates/worker-role.tpl")}" vars { - worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element(aws_iam_instance_profile.workers_launch_template.*.role, count.index)}" + worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element(coalescelist(aws_iam_instance_profile.workers_launch_template.*.role, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name), count.index)}" } } @@ -48,7 +48,7 @@ data "template_file" "worker_role_arns" { template = "${file("${path.module}/templates/worker-role.tpl")}" vars { - worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element(aws_iam_instance_profile.workers.*.role, count.index)}" + worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element(coalescelist(aws_iam_instance_profile.workers.*.role, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name), count.index)}" } } diff --git a/cluster.tf b/cluster.tf index 003e591b41..5877c627a0 100644 --- a/cluster.tf +++ b/cluster.tf @@ -1,7 +1,7 @@ resource "aws_eks_cluster" "this" { name = "${var.cluster_name}" enabled_cluster_log_types = "${var.cluster_enabled_log_types}" - role_arn = "${aws_iam_role.cluster.arn}" + role_arn = "${local.cluster_iam_role_arn}" version = "${var.cluster_version}" vpc_config { @@ -58,14 +58,17 @@ resource "aws_iam_role" "cluster" { permissions_boundary = "${var.permissions_boundary}" path = "${var.iam_path}" force_detach_policies = true + count = "${var.manage_cluster_iam_resources ? 1 : 0}" } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" role = "${aws_iam_role.cluster.name}" + count = "${var.manage_cluster_iam_resources ? 1 : 0}" } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" role = "${aws_iam_role.cluster.name}" + count = "${var.manage_cluster_iam_resources ? 1 : 0}" } diff --git a/data.tf b/data.tf index e0e664a9e2..72d2a8173a 100644 --- a/data.tf +++ b/data.tf @@ -100,3 +100,18 @@ data "template_file" "launch_template_userdata" { kubelet_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "kubelet_extra_args", local.workers_group_launch_template_defaults["kubelet_extra_args"])}" } } + +data "aws_iam_role" "custom_cluster_iam_role" { + name = "${var.cluster_iam_role_name}" + count = "${var.manage_cluster_iam_resources ? 0 : 1}" +} + +data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" { + name = "${lookup(var.worker_groups[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}" + count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_count}" +} + +data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instance_profile" { + name = "${lookup(var.worker_groups_launch_template[count.index], "iam_instance_profile_name", local.workers_group_launch_template_defaults["iam_instance_profile_name"])}" + count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_launch_template_count}" +} diff --git a/local.tf b/local.tf index 3f6149d05c..e87c1fe8ec 100644 --- a/local.tf +++ b/local.tf @@ -5,6 +5,9 @@ locals { # to workaround terraform not supporting short circut evaluation cluster_security_group_id = "${coalesce(join("", aws_security_group.cluster.*.id), var.cluster_security_group_id)}" + cluster_iam_role_name = "${coalesce(join("", aws_iam_role.cluster.*.name), var.cluster_iam_role_name)}" + cluster_iam_role_arn = "${coalesce(join("", aws_iam_role.cluster.*.arn), join("", data.aws_iam_role.custom_cluster_iam_role.*.arn))}" + worker_security_group_id = "${coalesce(join("", aws_security_group.workers.*.id), var.worker_security_group_id)}" default_iam_role_id = "${element(concat(aws_iam_role.workers.*.id, list("")), 0)}" kubeconfig_name = "${var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name}" @@ -34,7 +37,8 @@ locals { autoscaling_enabled = false # Sets whether policy and matching tags will be added to allow autoscaling. additional_security_group_ids = "" # A comma delimited list of additional security group ids to include in worker launch config protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. - iam_role_id = "${local.default_iam_role_id}" # Use the specified IAM role if set. + iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. + iam_role_id = "${local.default_iam_role_id}" # A custom IAM role id. Incompatible with iam_instance_profile_name. suspended_processes = "AZRebalance" # A comma delimited string of processes to to suspend. i.e. AZRebalance,HealthCheck,ReplaceUnhealthy target_group_arns = "" # A comma delimited list of ALB target group ARNs to be associated to the ASG enabled_metrics = "" # A comma delimited list of metrics to be collected i.e. GroupMinSize,GroupMaxSize,GroupDesiredCapacity @@ -79,7 +83,8 @@ locals { autoscaling_enabled = false # Sets whether policy and matching tags will be added to allow autoscaling. additional_security_group_ids = "" # A comma delimited list of additional security group ids to include in worker launch config protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. - iam_role_id = "${local.default_iam_role_id}" # Use the specified IAM role if set. + iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. + iam_role_id = "${local.default_iam_role_id}" # A custom IAM role id. Incompatible with iam_instance_profile_name. suspended_processes = "AZRebalance" # A comma delimited string of processes to to suspend. i.e. AZRebalance,HealthCheck,ReplaceUnhealthy target_group_arns = "" # A comma delimited list of ALB target group ARNs to be associated to the ASG enabled_metrics = "" # A comma delimited list of metrics to be collected i.e. GroupMinSize,GroupMaxSize,GroupDesiredCapacity diff --git a/outputs.tf b/outputs.tf index 2bca30937f..c99c58de45 100644 --- a/outputs.tf +++ b/outputs.tf @@ -35,12 +35,12 @@ output "config_map_aws_auth" { output "cluster_iam_role_name" { description = "IAM role name of the EKS cluster." - value = "${aws_iam_role.cluster.name}" + value = "${local.cluster_iam_role_name}" } output "cluster_iam_role_arn" { description = "IAM role ARN of the EKS cluster." - value = "${aws_iam_role.cluster.arn}" + value = "${local.cluster_iam_role_arn}" } output "kubeconfig" { @@ -105,10 +105,10 @@ output "worker_iam_instance_profile_names" { output "worker_iam_role_name" { description = "default IAM role name for EKS worker groups" - value = "${aws_iam_role.workers.name}" + value = "${element(coalescelist(aws_iam_role.workers.*.name, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name), 0)}" } output "worker_iam_role_arn" { description = "default IAM role ARN for EKS worker groups" - value = "${aws_iam_role.workers.arn}" + value = "${element(coalescelist(aws_iam_role.workers.*.arn, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_arn, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_arn), 0)}" } diff --git a/variables.tf b/variables.tf index 9c35afef4a..94c13237d4 100644 --- a/variables.tf +++ b/variables.tf @@ -257,3 +257,18 @@ variable "cluster_endpoint_public_access" { description = "Indicates whether or not the Amazon EKS public API server endpoint is enabled." default = true } + +variable "manage_cluster_iam_resources" { + description = "Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified." + default = true +} + +variable "cluster_iam_role_name" { + description = "IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false." + default = "" +} + +variable "manage_worker_iam_resources" { + description = "Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers." + default = true +} diff --git a/workers.tf b/workers.tf index 7790c62da5..5a68615e81 100644 --- a/workers.tf +++ b/workers.tf @@ -38,7 +38,7 @@ resource "aws_launch_configuration" "workers" { name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}" associate_public_ip_address = "${lookup(var.worker_groups[count.index], "public_ip", local.workers_group_defaults["public_ip"])}" security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",",lookup(var.worker_groups[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"] - iam_instance_profile = "${element(aws_iam_instance_profile.workers.*.id, count.index)}" + iam_instance_profile = "${element(coalescelist(aws_iam_instance_profile.workers.*.id, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.name), count.index)}" image_id = "${lookup(var.worker_groups[count.index], "ami_id", local.workers_group_defaults["ami_id"])}" instance_type = "${lookup(var.worker_groups[count.index], "instance_type", local.workers_group_defaults["instance_type"])}" key_name = "${lookup(var.worker_groups[count.index], "key_name", local.workers_group_defaults["key_name"])}" @@ -131,32 +131,36 @@ resource "aws_iam_role" "workers" { permissions_boundary = "${var.permissions_boundary}" path = "${var.iam_path}" force_detach_policies = true + count = "${var.manage_worker_iam_resources ? 1 : 0}" } resource "aws_iam_instance_profile" "workers" { name_prefix = "${aws_eks_cluster.this.name}" role = "${lookup(var.worker_groups[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" - count = "${var.worker_group_count}" + count = "${var.manage_worker_iam_resources ? var.worker_group_count : 0}" path = "${var.iam_path}" } resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" role = "${aws_iam_role.workers.name}" + count = "${var.manage_worker_iam_resources ? 1 : 0}" } resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" role = "${aws_iam_role.workers.name}" + count = "${var.manage_worker_iam_resources ? 1 : 0}" } resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" { policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" role = "${aws_iam_role.workers.name}" + count = "${var.manage_worker_iam_resources ? 1 : 0}" } resource "aws_iam_role_policy_attachment" "workers_additional_policies" { - count = "${var.workers_additional_policies_count}" + count = "${var.manage_worker_iam_resources ? var.workers_additional_policies_count : 0}" role = "${aws_iam_role.workers.name}" policy_arn = "${var.workers_additional_policies[count.index]}" } @@ -174,6 +178,7 @@ resource "null_resource" "tags_as_list_of_maps" { resource "aws_iam_role_policy_attachment" "workers_autoscaling" { policy_arn = "${aws_iam_policy.worker_autoscaling.arn}" role = "${aws_iam_role.workers.name}" + count = "${var.manage_worker_iam_resources ? 1 : 0}" } resource "aws_iam_policy" "worker_autoscaling" { @@ -181,6 +186,7 @@ resource "aws_iam_policy" "worker_autoscaling" { description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this.name}" policy = "${data.aws_iam_policy_document.worker_autoscaling.json}" path = "${var.iam_path}" + count = "${var.manage_worker_iam_resources ? 1 : 0}" } data "aws_iam_policy_document" "worker_autoscaling" { diff --git a/workers_launch_template.tf b/workers_launch_template.tf index c28ca741bc..73b31bcc96 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -75,7 +75,7 @@ resource "aws_launch_template" "workers_launch_template" { } iam_instance_profile { - name = "${element(aws_iam_instance_profile.workers_launch_template.*.name, count.index)}" + name = "${element(coalescelist(aws_iam_instance_profile.workers_launch_template.*.name, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.name), count.index)}" } image_id = "${lookup(var.worker_groups_launch_template[count.index], "ami_id", local.workers_group_launch_template_defaults["ami_id"])}" @@ -118,6 +118,6 @@ resource "aws_launch_template" "workers_launch_template" { resource "aws_iam_instance_profile" "workers_launch_template" { name_prefix = "${aws_eks_cluster.this.name}" role = "${lookup(var.worker_groups_launch_template[count.index], "iam_role_id", lookup(local.workers_group_launch_template_defaults, "iam_role_id"))}" - count = "${var.worker_group_launch_template_count}" + count = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_count : 0}" path = "${var.iam_path}" } From 2439c25771c9ec6450b8fbedc4b5db56549dbcac Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 7 May 2019 15:41:44 +0200 Subject: [PATCH 008/962] Cleaning up and deduplicating launch template related code (#370) * cleaning up launch template code * update changelog * Remove variable worker_group_launch_template_tags * misc fixes * fixing and simplifying example --- CHANGELOG.md | 1 + README.md | 5 +- data.tf | 10 ++-- examples/eks_test_fixture/main.tf | 21 +++---- local.tf | 57 ++++--------------- variables.tf | 15 ----- workers.tf | 27 ++++----- workers_launch_template.tf | 93 ++++++++++++------------------- 8 files changed, 75 insertions(+), 154 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41af2c1439..e01d73ed68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Set default suspended processes for ASG to `AZRebalance` (by @max-rocket-internet) - 4 small changes to `aws_launch_template` resource (by @max-rocket-internet) + - (Breaking Change) Rewritten and de-duplicated code related to Launch Templates (by @max-rocket-internet) - Add .prettierignore file (by @rothandrew) - Switch to https for the pre-commit repos (by @rothandrew) - Add instructions on how to enable the docker bridge network (by @rothandrew) diff --git a/README.md b/README.md index cd3e9e18a8..b1875a31f2 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | string | `"true"` | no | | worker\_group\_count | The number of maps contained within the worker_groups list. | string | `"1"` | no | | worker\_group\_launch\_template\_count | The number of maps contained within the worker_groups_launch_template list. | string | `"0"` | no | -| worker\_group\_launch\_template\_tags | A map defining extra tags to be applied to the worker group template ASG. | map | `{ "default": [] }` | no | | worker\_group\_tags | A map defining extra tags to be applied to the worker group ASG. | map | `{ "default": [] }` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | @@ -157,7 +156,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | workers\_additional\_policies | Additional policies to be added to workers | list | `[]` | no | | workers\_additional\_policies\_count | | string | `"0"` | no | | workers\_group\_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | map | `{}` | no | -| workers\_group\_launch\_template\_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | map | `{}` | no | | write\_aws\_auth\_config | Whether to write the aws-auth configmap file. | string | `"true"` | no | | write\_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | string | `"true"` | no | @@ -165,6 +163,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | |------|-------------| +| cluster\_arn | The Amazon Resource Name (ARN) of the cluster. | | cluster\_certificate\_authority\_data | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | | cluster\_endpoint | The endpoint for your EKS Kubernetes API. | | cluster\_iam\_role\_arn | IAM role ARN of the EKS cluster. | @@ -183,9 +182,9 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | workers\_asg\_arns | IDs of the autoscaling groups containing workers. | | workers\_asg\_names | Names of the autoscaling groups containing workers. | | workers\_default\_ami\_id | ID of the default worker group AMI | -| workers\_user\_data | User data of worker groups | | workers\_launch\_template\_arns | ARNs of the worker launch templates. | | workers\_launch\_template\_ids | IDs of the worker launch templates. | | workers\_launch\_template\_latest\_versions | Latest versions of the worker launch templates. | +| workers\_user\_data | User data of worker groups | diff --git a/data.tf b/data.tf index 72d2a8173a..5ef35bdfe8 100644 --- a/data.tf +++ b/data.tf @@ -94,10 +94,10 @@ data "template_file" "launch_template_userdata" { cluster_name = "${aws_eks_cluster.this.name}" endpoint = "${aws_eks_cluster.this.endpoint}" cluster_auth_base64 = "${aws_eks_cluster.this.certificate_authority.0.data}" - pre_userdata = "${lookup(var.worker_groups_launch_template[count.index], "pre_userdata", local.workers_group_launch_template_defaults["pre_userdata"])}" - additional_userdata = "${lookup(var.worker_groups_launch_template[count.index], "additional_userdata", local.workers_group_launch_template_defaults["additional_userdata"])}" - bootstrap_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "bootstrap_extra_args", local.workers_group_launch_template_defaults["bootstrap_extra_args"])}" - kubelet_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "kubelet_extra_args", local.workers_group_launch_template_defaults["kubelet_extra_args"])}" + pre_userdata = "${lookup(var.worker_groups_launch_template[count.index], "pre_userdata", local.workers_group_defaults["pre_userdata"])}" + additional_userdata = "${lookup(var.worker_groups_launch_template[count.index], "additional_userdata", local.workers_group_defaults["additional_userdata"])}" + bootstrap_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "bootstrap_extra_args", local.workers_group_defaults["bootstrap_extra_args"])}" + kubelet_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "kubelet_extra_args", local.workers_group_defaults["kubelet_extra_args"])}" } } @@ -112,6 +112,6 @@ data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" { } data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instance_profile" { - name = "${lookup(var.worker_groups_launch_template[count.index], "iam_instance_profile_name", local.workers_group_launch_template_defaults["iam_instance_profile_name"])}" + name = "${lookup(var.worker_groups_launch_template[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}" count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_launch_template_count}" } diff --git a/examples/eks_test_fixture/main.tf b/examples/eks_test_fixture/main.tf index bd1ea66732..058c1e3346 100644 --- a/examples/eks_test_fixture/main.tf +++ b/examples/eks_test_fixture/main.tf @@ -61,24 +61,17 @@ locals { worker_groups = [ { - # This will launch an autoscaling group with only On-Demand instances instance_type = "t2.small" additional_userdata = "echo foo bar" - subnets = "${join(",", module.vpc.private_subnets)}" - asg_desired_capacity = "2" + asg_desired_capacity = 2 }, ] worker_groups_launch_template = [ { - # This will launch an autoscaling group with only Spot Fleet instances - instance_type = "t2.small" - additional_userdata = "echo foo bar" - subnets = "${join(",", module.vpc.private_subnets)}" - additional_security_group_ids = "${aws_security_group.worker_group_mgmt_one.id},${aws_security_group.worker_group_mgmt_two.id}" - override_instance_type = "t3.small" - asg_desired_capacity = "2" - spot_instance_pools = 10 - on_demand_percentage_above_base_capacity = "0" + instance_type = "t2.small" + additional_userdata = "echo foo bar" + additional_security_group_ids = "${aws_security_group.worker_group_mgmt_one.id},${aws_security_group.worker_group_mgmt_two.id}" + asg_desired_capacity = 2 }, ] tags = { @@ -163,8 +156,8 @@ module "eks" { vpc_id = "${module.vpc.vpc_id}" worker_groups = "${local.worker_groups}" worker_groups_launch_template = "${local.worker_groups_launch_template}" - worker_group_count = "1" - worker_group_launch_template_count = "1" + worker_group_count = 1 + worker_group_launch_template_count = 1 worker_additional_security_group_ids = ["${aws_security_group.all_worker_mgmt.id}"] map_roles = "${var.map_roles}" map_roles_count = "${var.map_roles_count}" diff --git a/local.tf b/local.tf index e87c1fe8ec..6efaeba293 100644 --- a/local.tf +++ b/local.tf @@ -43,56 +43,19 @@ locals { target_group_arns = "" # A comma delimited list of ALB target group ARNs to be associated to the ASG enabled_metrics = "" # A comma delimited list of metrics to be collected i.e. GroupMinSize,GroupMaxSize,GroupDesiredCapacity placement_group = "" # The name of the placement group into which to launch the instances, if any. - } - - workers_group_defaults = "${merge(local.workers_group_defaults_defaults, var.workers_group_defaults)}" + service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS - workers_group_launch_template_defaults_defaults = { - name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. - ami_id = "${data.aws_ami.eks_worker.id}" # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI. - root_block_device_id = "${data.aws_ami.eks_worker.root_device_name}" # Root device name for workers. If non is provided, will assume default AMI was used. - asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group. - asg_max_size = "3" # Maximum worker capacity in the autoscaling group. - asg_min_size = "1" # Minimum worker capacity in the autoscaling group. - asg_force_delete = false # Enable forced deletion for the autoscaling group. - instance_type = "m4.large" # Size of the workers instances. - override_instance_type = "t3.large" # Need to specify at least one additional instance type for mixed instances policy. The instance_type holds higher priority for on demand instances. - on_demand_allocation_strategy = "prioritized" # Strategy to use when launching on-demand instances. Valid values: prioritized. - on_demand_base_capacity = "0" # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances - on_demand_percentage_above_base_capacity = "100" # Percentage split between on-demand and Spot instances above the base on-demand capacity - spot_allocation_strategy = "lowest-price" # The only valid value is lowest-price, which is also the default value. The Auto Scaling group selects the cheapest Spot pools and evenly allocates your Spot capacity across the number of Spot pools that you specify. - spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." - spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price - spot_price = "" # Cost of spot instance. - placement_tenancy = "default" # The tenancy of the instance. Valid values are "default" or "dedicated". - root_volume_size = "100" # root volume size of workers instances. - root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp2', or 'io1' - root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". - root_encrypted = "" # root volume encryption for workers. - kms_key_id = "" # KMS key ID used for encrypted block device. ASG must have access to this key. If not specified, the default KMS key will be used. - key_name = "" # The key name that should be used for the instances in the autoscaling group - pre_userdata = "" # userdata to pre-append to the default userdata. - bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI. - additional_userdata = "" # userdata to append to the default userdata. - ebs_optimized = true # sets whether to use ebs optimization on supported types. - enable_monitoring = true # Enables/disables detailed monitoring. - public_ip = false # Associate a public ip address with a worker - eni_delete = true # Delete the ENI on termination (if set to false you will have to manually delete before destroying) - kubelet_extra_args = "" # This string is passed directly to kubelet if set. Useful for adding labels or taints. - subnets = "${join(",", var.subnets)}" # A comma delimited string of subnets to place the worker nodes in. i.e. subnet-123,subnet-456,subnet-789 - autoscaling_enabled = false # Sets whether policy and matching tags will be added to allow autoscaling. - additional_security_group_ids = "" # A comma delimited list of additional security group ids to include in worker launch config - protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. - iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. - iam_role_id = "${local.default_iam_role_id}" # A custom IAM role id. Incompatible with iam_instance_profile_name. - suspended_processes = "AZRebalance" # A comma delimited string of processes to to suspend. i.e. AZRebalance,HealthCheck,ReplaceUnhealthy - target_group_arns = "" # A comma delimited list of ALB target group ARNs to be associated to the ASG - enabled_metrics = "" # A comma delimited list of metrics to be collected i.e. GroupMinSize,GroupMaxSize,GroupDesiredCapacity - placement_group = "" # The name of the placement group into which to launch the instances, if any. - service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS + # Settings for launch templates + root_block_device_name = "${data.aws_ami.eks_worker.root_device_name}" # Root device name for workers. If non is provided, will assume default AMI was used. + root_kms_key_id = "" # The KMS key to use when encrypting the root storage device + launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group + launch_template_placement_tenancy = "default" # The placement tenancy for instances + launch_template_placement_group = "" # The name of the placement group into which to launch the instances, if any. + root_encrypted = "" # Whether the volume should be encrypted or not + eni_delete = true # Delete the ENI on termination (if set to false you will have to manually delete before destroying) } - workers_group_launch_template_defaults = "${merge(local.workers_group_launch_template_defaults_defaults, var.workers_group_launch_template_defaults)}" + workers_group_defaults = "${merge(local.workers_group_defaults_defaults, var.workers_group_defaults)}" ebs_optimized = { "c1.medium" = false diff --git a/variables.tf b/variables.tf index 94c13237d4..c14f6609ae 100644 --- a/variables.tf +++ b/variables.tf @@ -138,21 +138,6 @@ variable "worker_group_launch_template_count" { default = "0" } -variable "workers_group_launch_template_defaults" { - description = "Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys." - type = "map" - default = {} -} - -variable "worker_group_launch_template_tags" { - description = "A map defining extra tags to be applied to the worker group template ASG." - type = "map" - - default = { - default = [] - } -} - variable "worker_security_group_id" { description = "If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster." default = "" diff --git a/workers.tf b/workers.tf index 5a68615e81..75e8580c11 100644 --- a/workers.tf +++ b/workers.tf @@ -1,19 +1,20 @@ # Worker Groups using Launch Configurations resource "aws_autoscaling_group" "workers" { - name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}" - desired_capacity = "${lookup(var.worker_groups[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}" - max_size = "${lookup(var.worker_groups[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}" - min_size = "${lookup(var.worker_groups[count.index], "asg_min_size", local.workers_group_defaults["asg_min_size"])}" - force_delete = "${lookup(var.worker_groups[count.index], "asg_force_delete", local.workers_group_defaults["asg_force_delete"])}" - target_group_arns = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "target_group_arns", ""), local.workers_group_defaults["target_group_arns"])))}"] - launch_configuration = "${element(aws_launch_configuration.workers.*.id, count.index)}" - vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_groups[count.index], "subnets", ""), local.workers_group_defaults["subnets"]))}"] - protect_from_scale_in = "${lookup(var.worker_groups[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}" - suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"] - enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"] - count = "${var.worker_group_count}" - placement_group = "${lookup(var.worker_groups[count.index], "placement_group", local.workers_group_defaults["placement_group"])}" + name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}" + desired_capacity = "${lookup(var.worker_groups[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}" + max_size = "${lookup(var.worker_groups[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}" + min_size = "${lookup(var.worker_groups[count.index], "asg_min_size", local.workers_group_defaults["asg_min_size"])}" + force_delete = "${lookup(var.worker_groups[count.index], "asg_force_delete", local.workers_group_defaults["asg_force_delete"])}" + target_group_arns = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "target_group_arns", ""), local.workers_group_defaults["target_group_arns"])))}"] + service_linked_role_arn = "${lookup(var.worker_groups[count.index], "service_linked_role_arn", local.workers_group_defaults["service_linked_role_arn"])}" + launch_configuration = "${element(aws_launch_configuration.workers.*.id, count.index)}" + vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_groups[count.index], "subnets", ""), local.workers_group_defaults["subnets"]))}"] + protect_from_scale_in = "${lookup(var.worker_groups[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}" + suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"] + enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"] + count = "${var.worker_group_count}" + placement_group = "${lookup(var.worker_groups[count.index], "placement_group", local.workers_group_defaults["placement_group"])}" tags = ["${concat( list( diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 73b31bcc96..99c26ddcb4 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -2,55 +2,34 @@ resource "aws_autoscaling_group" "workers_launch_template" { name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}" - desired_capacity = "${lookup(var.worker_groups_launch_template[count.index], "asg_desired_capacity", local.workers_group_launch_template_defaults["asg_desired_capacity"])}" - max_size = "${lookup(var.worker_groups_launch_template[count.index], "asg_max_size", local.workers_group_launch_template_defaults["asg_max_size"])}" - min_size = "${lookup(var.worker_groups_launch_template[count.index], "asg_min_size", local.workers_group_launch_template_defaults["asg_min_size"])}" - force_delete = "${lookup(var.worker_groups_launch_template[count.index], "asg_force_delete", local.workers_group_launch_template_defaults["asg_force_delete"])}" - target_group_arns = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "target_group_arns", ""), local.workers_group_launch_template_defaults["target_group_arns"])))}"] - service_linked_role_arn = "${lookup(var.worker_groups_launch_template[count.index], "service_linked_role_arn", local.workers_group_launch_template_defaults["service_linked_role_arn"])}" - - mixed_instances_policy { - instances_distribution { - on_demand_allocation_strategy = "${lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_launch_template_defaults["on_demand_allocation_strategy"])}" - on_demand_base_capacity = "${lookup(var.worker_groups_launch_template[count.index], "on_demand_base_capacity", local.workers_group_launch_template_defaults["on_demand_base_capacity"])}" - on_demand_percentage_above_base_capacity = "${lookup(var.worker_groups_launch_template[count.index], "on_demand_percentage_above_base_capacity", local.workers_group_launch_template_defaults["on_demand_percentage_above_base_capacity"])}" - spot_allocation_strategy = "${lookup(var.worker_groups_launch_template[count.index], "spot_allocation_strategy", local.workers_group_launch_template_defaults["spot_allocation_strategy"])}" - spot_instance_pools = "${lookup(var.worker_groups_launch_template[count.index], "spot_instance_pools", local.workers_group_launch_template_defaults["spot_instance_pools"])}" - spot_max_price = "${lookup(var.worker_groups_launch_template[count.index], "spot_max_price", local.workers_group_launch_template_defaults["spot_max_price"])}" - } - - launch_template { - launch_template_specification { - launch_template_id = "${element(aws_launch_template.workers_launch_template.*.id, count.index)}" - version = "$Latest" - } - - override { - instance_type = "${lookup(var.worker_groups_launch_template[count.index], "instance_type", local.workers_group_launch_template_defaults["instance_type"])}" - } - - override { - instance_type = "${lookup(var.worker_groups_launch_template[count.index], "override_instance_type", local.workers_group_launch_template_defaults["override_instance_type"])}" - } - } + desired_capacity = "${lookup(var.worker_groups_launch_template[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}" + max_size = "${lookup(var.worker_groups_launch_template[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}" + min_size = "${lookup(var.worker_groups_launch_template[count.index], "asg_min_size", local.workers_group_defaults["asg_min_size"])}" + force_delete = "${lookup(var.worker_groups_launch_template[count.index], "asg_force_delete", local.workers_group_defaults["asg_force_delete"])}" + target_group_arns = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "target_group_arns", ""), local.workers_group_defaults["target_group_arns"])))}"] + service_linked_role_arn = "${lookup(var.worker_groups_launch_template[count.index], "service_linked_role_arn", local.workers_group_defaults["service_linked_role_arn"])}" + vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "subnets", ""), local.workers_group_defaults["subnets"]))}"] + protect_from_scale_in = "${lookup(var.worker_groups_launch_template[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}" + suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"] + enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"] + count = "${var.worker_group_launch_template_count}" + placement_group = "${lookup(var.worker_groups_launch_template[count.index], "placement_group", local.workers_group_defaults["placement_group"])}" + + launch_template { + id = "${element(aws_launch_template.workers_launch_template.*.id, count.index)}" + version = "${lookup(var.worker_groups_launch_template[count.index], "launch_template_version", local.workers_group_defaults["launch_template_version"])}" } - vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "subnets", ""), local.workers_group_launch_template_defaults["subnets"]))}"] - protect_from_scale_in = "${lookup(var.worker_groups_launch_template[count.index], "protect_from_scale_in", local.workers_group_launch_template_defaults["protect_from_scale_in"])}" - suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "suspended_processes", ""), local.workers_group_launch_template_defaults["suspended_processes"])))}"] - enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "enabled_metrics", ""), local.workers_group_launch_template_defaults["enabled_metrics"])))}"] - count = "${var.worker_group_launch_template_count}" - tags = ["${concat( list( map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true), map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true), - map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups_launch_template[count.index], "autoscaling_enabled", local.workers_group_launch_template_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled" }", "value", "true", "propagate_at_launch", false), + map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups_launch_template[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled" }", "value", "true", "propagate_at_launch", false), map("key", "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}", "value", "", "propagate_at_launch", false), - map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_groups_launch_template[count.index], "root_volume_size", local.workers_group_launch_template_defaults["root_volume_size"])}Gi", "propagate_at_launch", false) + map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_groups_launch_template[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}Gi", "propagate_at_launch", false) ), local.asg_tags, - var.worker_group_launch_template_tags[contains(keys(var.worker_group_launch_template_tags), "${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}") ? "${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}" : "default"]) + var.worker_group_tags[contains(keys(var.worker_group_tags), "${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}") ? "${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}" : "default"]) }"] lifecycle { @@ -64,13 +43,13 @@ resource "aws_launch_template" "workers_launch_template" { name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}" network_interfaces { - associate_public_ip_address = "${lookup(var.worker_groups_launch_template[count.index], "public_ip", local.workers_group_launch_template_defaults["public_ip"])}" - delete_on_termination = "${lookup(var.worker_groups_launch_template[count.index], "eni_delete", local.workers_group_launch_template_defaults["eni_delete"])}" + associate_public_ip_address = "${lookup(var.worker_groups_launch_template[count.index], "public_ip", local.workers_group_defaults["public_ip"])}" + delete_on_termination = "${lookup(var.worker_groups_launch_template[count.index], "eni_delete", local.workers_group_defaults["eni_delete"])}" security_groups = [ "${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", - "${compact(split(",",lookup(var.worker_groups_launch_template[count.index],"additional_security_group_ids", local.workers_group_launch_template_defaults["additional_security_group_ids"])))}", + "${compact(split(",",lookup(var.worker_groups_launch_template[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}", ] } @@ -78,19 +57,19 @@ resource "aws_launch_template" "workers_launch_template" { name = "${element(coalescelist(aws_iam_instance_profile.workers_launch_template.*.name, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.name), count.index)}" } - image_id = "${lookup(var.worker_groups_launch_template[count.index], "ami_id", local.workers_group_launch_template_defaults["ami_id"])}" - instance_type = "${lookup(var.worker_groups_launch_template[count.index], "instance_type", local.workers_group_launch_template_defaults["instance_type"])}" - key_name = "${lookup(var.worker_groups_launch_template[count.index], "key_name", local.workers_group_launch_template_defaults["key_name"])}" + image_id = "${lookup(var.worker_groups_launch_template[count.index], "ami_id", local.workers_group_defaults["ami_id"])}" + instance_type = "${lookup(var.worker_groups_launch_template[count.index], "instance_type", local.workers_group_defaults["instance_type"])}" + key_name = "${lookup(var.worker_groups_launch_template[count.index], "key_name", local.workers_group_defaults["key_name"])}" user_data = "${base64encode(element(data.template_file.launch_template_userdata.*.rendered, count.index))}" - ebs_optimized = "${lookup(var.worker_groups_launch_template[count.index], "ebs_optimized", lookup(local.ebs_optimized, lookup(var.worker_groups_launch_template[count.index], "instance_type", local.workers_group_launch_template_defaults["instance_type"]), false))}" + ebs_optimized = "${lookup(var.worker_groups_launch_template[count.index], "ebs_optimized", lookup(local.ebs_optimized, lookup(var.worker_groups_launch_template[count.index], "instance_type", local.workers_group_defaults["instance_type"]), false))}" monitoring { - enabled = "${lookup(var.worker_groups_launch_template[count.index], "enable_monitoring", local.workers_group_launch_template_defaults["enable_monitoring"])}" + enabled = "${lookup(var.worker_groups_launch_template[count.index], "enable_monitoring", local.workers_group_defaults["enable_monitoring"])}" } placement { - tenancy = "${lookup(var.worker_groups_launch_template[count.index], "placement_tenancy", local.workers_group_launch_template_defaults["placement_tenancy"])}" - group_name = "${lookup(var.worker_groups_launch_template[count.index], "placement_group", local.workers_group_launch_template_defaults["placement_group"])}" + tenancy = "${lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_tenancy", local.workers_group_defaults["launch_template_placement_tenancy"])}" + group_name = "${lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])}" } count = "${var.worker_group_launch_template_count}" @@ -100,14 +79,14 @@ resource "aws_launch_template" "workers_launch_template" { } block_device_mappings { - device_name = "${data.aws_ami.eks_worker.root_device_name}" + device_name = "${lookup(var.worker_groups_launch_template[count.index], "root_block_device_name", local.workers_group_defaults["root_block_device_name"])}" ebs { - volume_size = "${lookup(var.worker_groups_launch_template[count.index], "root_volume_size", local.workers_group_launch_template_defaults["root_volume_size"])}" - volume_type = "${lookup(var.worker_groups_launch_template[count.index], "root_volume_type", local.workers_group_launch_template_defaults["root_volume_type"])}" - iops = "${lookup(var.worker_groups_launch_template[count.index], "root_iops", local.workers_group_launch_template_defaults["root_iops"])}" - encrypted = "${lookup(var.worker_groups_launch_template[count.index], "root_encrypted", local.workers_group_launch_template_defaults["root_encrypted"])}" - kms_key_id = "${lookup(var.worker_groups_launch_template[count.index], "kms_key_id", local.workers_group_launch_template_defaults["kms_key_id"])}" + volume_size = "${lookup(var.worker_groups_launch_template[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}" + volume_type = "${lookup(var.worker_groups_launch_template[count.index], "root_volume_type", local.workers_group_defaults["root_volume_type"])}" + iops = "${lookup(var.worker_groups_launch_template[count.index], "root_iops", local.workers_group_defaults["root_iops"])}" + encrypted = "${lookup(var.worker_groups_launch_template[count.index], "root_encrypted", local.workers_group_defaults["root_encrypted"])}" + kms_key_id = "${lookup(var.worker_groups_launch_template[count.index], "root_kms_key_id", local.workers_group_defaults["root_kms_key_id"])}" delete_on_termination = true } } @@ -117,7 +96,7 @@ resource "aws_launch_template" "workers_launch_template" { resource "aws_iam_instance_profile" "workers_launch_template" { name_prefix = "${aws_eks_cluster.this.name}" - role = "${lookup(var.worker_groups_launch_template[count.index], "iam_role_id", lookup(local.workers_group_launch_template_defaults, "iam_role_id"))}" + role = "${lookup(var.worker_groups_launch_template[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" count = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_count : 0}" path = "${var.iam_path}" } From ae2f8e58dbe1fc1bec1dbd6861a3fd2bc648f548 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 7 May 2019 16:50:42 +0200 Subject: [PATCH 009/962] Adding new mixed type of worker group with instance overrides and mixed instances policy (#371) * Adding new mixed type of worker group with instance overrides and mixed instances policy * moving all count and lifecycle rule parameters to top/bottom * adding custom IAM parts * updating doc with new options * fixes for spot instances --- CHANGELOG.md | 3 +- README.md | 2 + aws_auth.tf | 5 +- cluster.tf | 12 +-- data.tf | 34 ++++++-- docs/spot-instances.md | 131 +++++++++++++++---------------- kubectl.tf | 2 +- local.tf | 12 +++ outputs.tf | 8 +- variables.tf | 17 ++++ workers.tf | 44 +++++------ workers_launch_template.tf | 18 ++--- workers_launch_template_mixed.tf | 122 ++++++++++++++++++++++++++++ 13 files changed, 289 insertions(+), 121 deletions(-) create mode 100644 workers_launch_template_mixed.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index e01d73ed68..374b54ae66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,10 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added support for custom service linked role for Auto Scaling group (by @voanhduy1512) - Added support for custom IAM roles for cluster and workers (by @erks) -- Add cluster arn to outputs (by @alexsn) +- Added cluster ARN to outputs (by @alexsn) - Added outputs for `workers_user_data` and `workers_default_ami_id` (by @max-rocket-internet) - Added doc about spot instances (by @max-rocket-internet) +- Added new worker group option with a mixed instances policy (by @max-rocket-internet) ### Changed diff --git a/README.md b/README.md index b1875a31f2..61dc4d182a 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | string | `"true"` | no | | worker\_group\_count | The number of maps contained within the worker_groups list. | string | `"1"` | no | | worker\_group\_launch\_template\_count | The number of maps contained within the worker_groups_launch_template list. | string | `"0"` | no | +| worker\_group\_launch\_template\_mixed | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | +| worker\_group\_launch\_template\_mixed\_count | The number of maps contained within the worker_group_launch_template_mixed list. | string | `"0"` | no | | worker\_group\_tags | A map defining extra tags to be applied to the worker group ASG. | map | `{ "default": [] }` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | diff --git a/aws_auth.tf b/aws_auth.tf index a86349c52c..a9fc899f26 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -1,10 +1,11 @@ resource "local_file" "config_map_aws_auth" { + count = "${var.write_aws_auth_config ? 1 : 0}" content = "${data.template_file.config_map_aws_auth.rendered}" filename = "${var.config_output_path}config-map-aws-auth_${var.cluster_name}.yaml" - count = "${var.write_aws_auth_config ? 1 : 0}" } resource "null_resource" "update_config_map_aws_auth" { + count = "${var.manage_aws_auth ? 1 : 0}" depends_on = ["aws_eks_cluster.this"] provisioner "local-exec" { @@ -28,8 +29,6 @@ EOS config_map_rendered = "${data.template_file.config_map_aws_auth.rendered}" endpoint = "${aws_eks_cluster.this.endpoint}" } - - count = "${var.manage_aws_auth ? 1 : 0}" } data "aws_caller_identity" "current" {} diff --git a/cluster.tf b/cluster.tf index 5877c627a0..dc789ee83e 100644 --- a/cluster.tf +++ b/cluster.tf @@ -23,14 +23,15 @@ resource "aws_eks_cluster" "this" { } resource "aws_security_group" "cluster" { + count = "${var.cluster_create_security_group ? 1 : 0}" name_prefix = "${var.cluster_name}" description = "EKS cluster security group." vpc_id = "${var.vpc_id}" tags = "${merge(var.tags, map("Name", "${var.cluster_name}-eks_cluster_sg"))}" - count = "${var.cluster_create_security_group ? 1 : 0}" } resource "aws_security_group_rule" "cluster_egress_internet" { + count = "${var.cluster_create_security_group ? 1 : 0}" description = "Allow cluster egress access to the Internet." protocol = "-1" security_group_id = "${aws_security_group.cluster.id}" @@ -38,10 +39,10 @@ resource "aws_security_group_rule" "cluster_egress_internet" { from_port = 0 to_port = 0 type = "egress" - count = "${var.cluster_create_security_group ? 1 : 0}" } resource "aws_security_group_rule" "cluster_https_worker_ingress" { + count = "${var.cluster_create_security_group ? 1 : 0}" description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" security_group_id = "${aws_security_group.cluster.id}" @@ -49,26 +50,25 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { from_port = 443 to_port = 443 type = "ingress" - count = "${var.cluster_create_security_group ? 1 : 0}" } resource "aws_iam_role" "cluster" { + count = "${var.manage_cluster_iam_resources ? 1 : 0}" name_prefix = "${var.cluster_name}" assume_role_policy = "${data.aws_iam_policy_document.cluster_assume_role_policy.json}" permissions_boundary = "${var.permissions_boundary}" path = "${var.iam_path}" force_detach_policies = true - count = "${var.manage_cluster_iam_resources ? 1 : 0}" } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" { + count = "${var.manage_cluster_iam_resources ? 1 : 0}" policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" role = "${aws_iam_role.cluster.name}" - count = "${var.manage_cluster_iam_resources ? 1 : 0}" } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { + count = "${var.manage_cluster_iam_resources ? 1 : 0}" policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" role = "${aws_iam_role.cluster.name}" - count = "${var.manage_cluster_iam_resources ? 1 : 0}" } diff --git a/data.tf b/data.tf index 5ef35bdfe8..91e6570172 100644 --- a/data.tf +++ b/data.tf @@ -58,13 +58,13 @@ data "template_file" "kubeconfig" { } data "template_file" "aws_authenticator_env_variables" { + count = "${length(var.kubeconfig_aws_authenticator_env_variables)}" + template = < Date: Tue, 7 May 2019 17:06:39 +0200 Subject: [PATCH 010/962] New release v4.0.0 (#373) --- CHANGELOG.md | 29 +++++++++++++++++++---------- version | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 374b54ae66..0cdbc292c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,19 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v3.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v3.0.0...HEAD)] - 2019-05-??] +## [[v4.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...HEAD)] - 2019-06-??] + +### Added + +- Write your awesome addition here (by @you) + +### Changed + +- Write your awesome change here (by @you) + +# History + +## [[v4.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v3.0.0...v4.0.0)] - 2019-05-07] ### Added @@ -20,15 +32,12 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed - - Set default suspended processes for ASG to `AZRebalance` (by @max-rocket-internet) - - 4 small changes to `aws_launch_template` resource (by @max-rocket-internet) - - (Breaking Change) Rewritten and de-duplicated code related to Launch Templates (by @max-rocket-internet) - - Add .prettierignore file (by @rothandrew) - - Switch to https for the pre-commit repos (by @rothandrew) - - Add instructions on how to enable the docker bridge network (by @rothandrew) - - Write your awesome change here (by @you) - -# History +- Set default suspended processes for ASG to `AZRebalance` (by @max-rocket-internet) +- 4 small changes to `aws_launch_template` resource (by @max-rocket-internet) +- (Breaking Change) Rewritten and de-duplicated code related to Launch Templates (by @max-rocket-internet) +- Add .prettierignore file (by @rothandrew) +- Switch to https for the pre-commit repos (by @rothandrew) +- Add instructions on how to enable the docker bridge network (by @rothandrew) ## [[v3.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.1...v3.0.0)] - 2019-04-15] diff --git a/version b/version index ad55eb85fc..857572fcd3 100644 --- a/version +++ b/version @@ -1 +1 @@ -v3.0.0 +v4.0.0 From f0838165e24f81ab880ee3d402a512b3371c2bfd Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 7 May 2019 18:22:39 +0200 Subject: [PATCH 011/962] Fix annoying typo: worker_group_xx vs worker_groups_xx (#374) --- CHANGELOG.md | 6 +++ README.md | 4 +- data.tf | 2 +- variables.tf | 4 +- version | 2 +- workers_launch_template_mixed.tf | 90 ++++++++++++++++---------------- 6 files changed, 57 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cdbc292c1..6fdb17b263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,12 @@ project adheres to [Semantic Versioning](http://semver.org/). # History +## [[v4.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1)] - 2019-05-07] + +### Changed + +- Fix annoying typo: worker_group_xx vs worker_groups_xx (by @max-rocket-internet) + ## [[v4.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v3.0.0...v4.0.0)] - 2019-05-07] ### Added diff --git a/README.md b/README.md index 61dc4d182a..33cc7e7146 100644 --- a/README.md +++ b/README.md @@ -148,11 +148,11 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | string | `"true"` | no | | worker\_group\_count | The number of maps contained within the worker_groups list. | string | `"1"` | no | | worker\_group\_launch\_template\_count | The number of maps contained within the worker_groups_launch_template list. | string | `"0"` | no | -| worker\_group\_launch\_template\_mixed | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | -| worker\_group\_launch\_template\_mixed\_count | The number of maps contained within the worker_group_launch_template_mixed list. | string | `"0"` | no | +| worker\_group\_launch\_template\_mixed\_count | The number of maps contained within the worker_groups_launch_template_mixed list. | string | `"0"` | no | | worker\_group\_tags | A map defining extra tags to be applied to the worker group ASG. | map | `{ "default": [] }` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | +| worker\_groups\_launch\_template\_mixed | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | | worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster. | string | `""` | no | | worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | string | `"1025"` | no | | workers\_additional\_policies | Additional policies to be added to workers | list | `[]` | no | diff --git a/data.tf b/data.tf index 91e6570172..e1ec701d2b 100644 --- a/data.tf +++ b/data.tf @@ -133,5 +133,5 @@ data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instanc data "aws_iam_instance_profile" "custom_worker_group_launch_template_mixed_iam_instance_profile" { count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_launch_template_mixed_count}" - name = "${lookup(var.worker_group_launch_template_mixed[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}" + name = "${lookup(var.worker_groups_launch_template_mixed[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}" } diff --git a/variables.tf b/variables.tf index c1c4529e1b..fb769c4180 100644 --- a/variables.tf +++ b/variables.tf @@ -132,7 +132,7 @@ variable "worker_groups_launch_template" { ] } -variable "worker_group_launch_template_mixed" { +variable "worker_groups_launch_template_mixed" { description = "A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys." type = "list" @@ -144,7 +144,7 @@ variable "worker_group_launch_template_mixed" { } variable "worker_group_launch_template_mixed_count" { - description = "The number of maps contained within the worker_group_launch_template_mixed list." + description = "The number of maps contained within the worker_groups_launch_template_mixed list." type = "string" default = "0" } diff --git a/version b/version index 857572fcd3..82f24fdfff 100644 --- a/version +++ b/version @@ -1 +1 @@ -v4.0.0 +v4.0.1 diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf index bac40b966e..3222f6125f 100644 --- a/workers_launch_template_mixed.tf +++ b/workers_launch_template_mixed.tf @@ -2,63 +2,63 @@ resource "aws_autoscaling_group" "workers_launch_template_mixed" { count = "${var.worker_group_launch_template_mixed_count}" - name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}" - desired_capacity = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}" - max_size = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}" - min_size = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_min_size", local.workers_group_defaults["asg_min_size"])}" - force_delete = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_force_delete", local.workers_group_defaults["asg_force_delete"])}" - target_group_arns = ["${compact(split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "target_group_arns", ""), local.workers_group_defaults["target_group_arns"])))}"] - service_linked_role_arn = "${lookup(var.worker_group_launch_template_mixed[count.index], "service_linked_role_arn", local.workers_group_defaults["service_linked_role_arn"])}" - vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "subnets", ""), local.workers_group_defaults["subnets"]))}"] - protect_from_scale_in = "${lookup(var.worker_group_launch_template_mixed[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}" - suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"] - enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"] - placement_group = "${lookup(var.worker_group_launch_template_mixed[count.index], "placement_group", local.workers_group_defaults["placement_group"])}" + name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}" + desired_capacity = "${lookup(var.worker_groups_launch_template_mixed[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}" + max_size = "${lookup(var.worker_groups_launch_template_mixed[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}" + min_size = "${lookup(var.worker_groups_launch_template_mixed[count.index], "asg_min_size", local.workers_group_defaults["asg_min_size"])}" + force_delete = "${lookup(var.worker_groups_launch_template_mixed[count.index], "asg_force_delete", local.workers_group_defaults["asg_force_delete"])}" + target_group_arns = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "target_group_arns", ""), local.workers_group_defaults["target_group_arns"])))}"] + service_linked_role_arn = "${lookup(var.worker_groups_launch_template_mixed[count.index], "service_linked_role_arn", local.workers_group_defaults["service_linked_role_arn"])}" + vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "subnets", ""), local.workers_group_defaults["subnets"]))}"] + protect_from_scale_in = "${lookup(var.worker_groups_launch_template_mixed[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}" + suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"] + enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"] + placement_group = "${lookup(var.worker_groups_launch_template_mixed[count.index], "placement_group", local.workers_group_defaults["placement_group"])}" mixed_instances_policy { instances_distribution { - on_demand_allocation_strategy = "${lookup(var.worker_group_launch_template_mixed[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"])}" - on_demand_base_capacity = "${lookup(var.worker_group_launch_template_mixed[count.index], "on_demand_base_capacity", local.workers_group_defaults["on_demand_base_capacity"])}" - on_demand_percentage_above_base_capacity = "${lookup(var.worker_group_launch_template_mixed[count.index], "on_demand_percentage_above_base_capacity", local.workers_group_defaults["on_demand_percentage_above_base_capacity"])}" - spot_allocation_strategy = "${lookup(var.worker_group_launch_template_mixed[count.index], "spot_allocation_strategy", local.workers_group_defaults["spot_allocation_strategy"])}" - spot_instance_pools = "${lookup(var.worker_group_launch_template_mixed[count.index], "spot_instance_pools", local.workers_group_defaults["spot_instance_pools"])}" - spot_max_price = "${lookup(var.worker_group_launch_template_mixed[count.index], "spot_max_price", local.workers_group_defaults["spot_max_price"])}" + on_demand_allocation_strategy = "${lookup(var.worker_groups_launch_template_mixed[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"])}" + on_demand_base_capacity = "${lookup(var.worker_groups_launch_template_mixed[count.index], "on_demand_base_capacity", local.workers_group_defaults["on_demand_base_capacity"])}" + on_demand_percentage_above_base_capacity = "${lookup(var.worker_groups_launch_template_mixed[count.index], "on_demand_percentage_above_base_capacity", local.workers_group_defaults["on_demand_percentage_above_base_capacity"])}" + spot_allocation_strategy = "${lookup(var.worker_groups_launch_template_mixed[count.index], "spot_allocation_strategy", local.workers_group_defaults["spot_allocation_strategy"])}" + spot_instance_pools = "${lookup(var.worker_groups_launch_template_mixed[count.index], "spot_instance_pools", local.workers_group_defaults["spot_instance_pools"])}" + spot_max_price = "${lookup(var.worker_groups_launch_template_mixed[count.index], "spot_max_price", local.workers_group_defaults["spot_max_price"])}" } launch_template { launch_template_specification { launch_template_id = "${element(aws_launch_template.workers_launch_template_mixed.*.id, count.index)}" - version = "${lookup(var.worker_group_launch_template_mixed[count.index], "launch_template_version", local.workers_group_defaults["launch_template_version"])}" + version = "${lookup(var.worker_groups_launch_template_mixed[count.index], "launch_template_version", local.workers_group_defaults["launch_template_version"])}" } override { - instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_1", local.workers_group_defaults["override_instance_type_1"])}" + instance_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "override_instance_type_1", local.workers_group_defaults["override_instance_type_1"])}" } override { - instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_2", local.workers_group_defaults["override_instance_type_2"])}" + instance_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "override_instance_type_2", local.workers_group_defaults["override_instance_type_2"])}" } override { - instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_3", local.workers_group_defaults["override_instance_type_3"])}" + instance_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "override_instance_type_3", local.workers_group_defaults["override_instance_type_3"])}" } override { - instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_4", local.workers_group_defaults["override_instance_type_4"])}" + instance_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "override_instance_type_4", local.workers_group_defaults["override_instance_type_4"])}" } } } tags = ["${concat( list( - map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true), + map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true), map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true), - map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_group_launch_template_mixed[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled" }", "value", "true", "propagate_at_launch", false), + map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups_launch_template_mixed[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled" }", "value", "true", "propagate_at_launch", false), map("key", "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}", "value", "", "propagate_at_launch", false), - map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_group_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}Gi", "propagate_at_launch", false) + map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}Gi", "propagate_at_launch", false) ), local.asg_tags, - var.worker_group_tags[contains(keys(var.worker_group_tags), "${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}") ? "${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}" : "default"]) + var.worker_group_tags[contains(keys(var.worker_group_tags), "${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}") ? "${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}" : "default"]) }"] lifecycle { @@ -69,42 +69,42 @@ resource "aws_autoscaling_group" "workers_launch_template_mixed" { resource "aws_launch_template" "workers_launch_template_mixed" { count = "${var.worker_group_launch_template_mixed_count}" - name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}" + name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}" network_interfaces { - associate_public_ip_address = "${lookup(var.worker_group_launch_template_mixed[count.index], "public_ip", local.workers_group_defaults["public_ip"])}" - delete_on_termination = "${lookup(var.worker_group_launch_template_mixed[count.index], "eni_delete", local.workers_group_defaults["eni_delete"])}" - security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",",lookup(var.worker_group_launch_template_mixed[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"] + associate_public_ip_address = "${lookup(var.worker_groups_launch_template_mixed[count.index], "public_ip", local.workers_group_defaults["public_ip"])}" + delete_on_termination = "${lookup(var.worker_groups_launch_template_mixed[count.index], "eni_delete", local.workers_group_defaults["eni_delete"])}" + security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",",lookup(var.worker_groups_launch_template_mixed[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"] } iam_instance_profile { name = "${element(coalescelist(aws_iam_instance_profile.workers_launch_template_mixed.*.name, data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.name), count.index)}" } - image_id = "${lookup(var.worker_group_launch_template_mixed[count.index], "ami_id", local.workers_group_defaults["ami_id"])}" - instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "instance_type", local.workers_group_defaults["instance_type"])}" - key_name = "${lookup(var.worker_group_launch_template_mixed[count.index], "key_name", local.workers_group_defaults["key_name"])}" + image_id = "${lookup(var.worker_groups_launch_template_mixed[count.index], "ami_id", local.workers_group_defaults["ami_id"])}" + instance_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "instance_type", local.workers_group_defaults["instance_type"])}" + key_name = "${lookup(var.worker_groups_launch_template_mixed[count.index], "key_name", local.workers_group_defaults["key_name"])}" user_data = "${base64encode(element(data.template_file.workers_launch_template_mixed.*.rendered, count.index))}" - ebs_optimized = "${lookup(var.worker_group_launch_template_mixed[count.index], "ebs_optimized", lookup(local.ebs_optimized, lookup(var.worker_group_launch_template_mixed[count.index], "instance_type", local.workers_group_defaults["instance_type"]), false))}" + ebs_optimized = "${lookup(var.worker_groups_launch_template_mixed[count.index], "ebs_optimized", lookup(local.ebs_optimized, lookup(var.worker_groups_launch_template_mixed[count.index], "instance_type", local.workers_group_defaults["instance_type"]), false))}" monitoring { - enabled = "${lookup(var.worker_group_launch_template_mixed[count.index], "enable_monitoring", local.workers_group_defaults["enable_monitoring"])}" + enabled = "${lookup(var.worker_groups_launch_template_mixed[count.index], "enable_monitoring", local.workers_group_defaults["enable_monitoring"])}" } placement { - tenancy = "${lookup(var.worker_group_launch_template_mixed[count.index], "launch_template_placement_tenancy", local.workers_group_defaults["launch_template_placement_tenancy"])}" - group_name = "${lookup(var.worker_group_launch_template_mixed[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])}" + tenancy = "${lookup(var.worker_groups_launch_template_mixed[count.index], "launch_template_placement_tenancy", local.workers_group_defaults["launch_template_placement_tenancy"])}" + group_name = "${lookup(var.worker_groups_launch_template_mixed[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])}" } block_device_mappings { - device_name = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_block_device_name", local.workers_group_defaults["root_block_device_name"])}" + device_name = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_block_device_name", local.workers_group_defaults["root_block_device_name"])}" ebs { - volume_size = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}" - volume_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_volume_type", local.workers_group_defaults["root_volume_type"])}" - iops = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_iops", local.workers_group_defaults["root_iops"])}" - encrypted = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_encrypted", local.workers_group_defaults["root_encrypted"])}" - kms_key_id = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_kms_key_id", local.workers_group_defaults["root_kms_key_id"])}" + volume_size = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}" + volume_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_volume_type", local.workers_group_defaults["root_volume_type"])}" + iops = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_iops", local.workers_group_defaults["root_iops"])}" + encrypted = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_encrypted", local.workers_group_defaults["root_encrypted"])}" + kms_key_id = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_kms_key_id", local.workers_group_defaults["root_kms_key_id"])}" delete_on_termination = true } } @@ -117,6 +117,6 @@ resource "aws_launch_template" "workers_launch_template_mixed" { resource "aws_iam_instance_profile" "workers_launch_template_mixed" { count = "${var.worker_group_launch_template_mixed_count}" name_prefix = "${aws_eks_cluster.this.name}" - role = "${lookup(var.worker_group_launch_template_mixed[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" + role = "${lookup(var.worker_groups_launch_template_mixed[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" path = "${var.iam_path}" } From d6fa9f48ff02b0d6ddb7e3ce17d2a5e5c71f535a Mon Sep 17 00:00:00 2001 From: Max Williams Date: Wed, 8 May 2019 15:11:05 +0200 Subject: [PATCH 012/962] Better examples, PR template changes, general tidy up (#375) * adding 3 examples * removing old example * updating PR template * fix this typo * update after renaming default example * add missing launch_template_mixed stuff to aws_auth * fix 2 examples with public subnets * update changelog for new minor release --- .github/PULL_REQUEST_TEMPLATE.md | 9 +- .kitchen.yml | 2 +- .travis.yml | 20 ++- CHANGELOG.md | 9 ++ README.md | 13 +- aws_auth.tf | 11 +- data.tf | 8 +- examples/README.md | 7 + examples/{eks_test_fixture => basic}/main.tf | 121 +++++++----------- .../{eks_test_fixture => basic}/outputs.tf | 2 +- .../{eks_test_fixture => basic}/variables.tf | 0 examples/eks_test_fixture/README.md | 117 ----------------- examples/launch_templates/main.tf | 60 +++++++++ examples/launch_templates/outputs.tf | 24 ++++ examples/launch_templates/variables.tf | 3 + examples/spot_instances/main.tf | 60 +++++++++ examples/spot_instances/outputs.tf | 24 ++++ examples/spot_instances/variables.tf | 3 + variables.tf | 6 +- version | 2 +- 20 files changed, 270 insertions(+), 231 deletions(-) create mode 100644 examples/README.md rename examples/{eks_test_fixture => basic}/main.tf (50%) rename examples/{eks_test_fixture => basic}/outputs.tf (88%) rename examples/{eks_test_fixture => basic}/variables.tf (100%) delete mode 100644 examples/eks_test_fixture/README.md create mode 100644 examples/launch_templates/main.tf create mode 100644 examples/launch_templates/outputs.tf create mode 100644 examples/launch_templates/variables.tf create mode 100644 examples/spot_instances/main.tf create mode 100644 examples/spot_instances/outputs.tf create mode 100644 examples/spot_instances/variables.tf diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5a32376b08..2d8f78be51 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,8 +6,7 @@ Please explain the changes you made here and link to any relevant issues. ### Checklist -- [ ] `terraform fmt` and `terraform validate` both work from the root and `examples/eks_test_fixture` directories (look in CI for an example) -- [ ] Tests for the changes have been added and passing (for bug fixes/features) -- [ ] Test results are pasted in this PR (in lieu of CI) -- [ ] I've added my change to CHANGELOG.md -- [ ] Any breaking changes are highlighted above +- [ ] `terraform fmt` and `terraform validate` both work from the root and `examples/*` directories +- [ ] CI tests are passing +- [ ] I've added my change to CHANGELOG.md and highlighted any breaking changes +- [ ] README.md has been updated after any changes to variables and outputs. See https://github.com/terraform-aws-modules/terraform-aws-eks/#doc-generation diff --git a/.kitchen.yml b/.kitchen.yml index 9f73ea39dd..b185344d90 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -1,7 +1,7 @@ --- driver: name: "terraform" - root_module_directory: "examples/eks_test_fixture" + root_module_directory: "examples/basic" provisioner: name: "terraform" diff --git a/.travis.yml b/.travis.yml index cdcc24b4c8..7954f76975 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,19 +34,15 @@ script: - terraform init - terraform fmt -check=true - terraform validate -var "region=${AWS_REGION}" -var "vpc_id=vpc-123456" -var "subnets=[\"subnet-12345a\"]" -var "workers_ami_id=ami-123456" -var "cluster_ingress_cidrs=[]" -var "cluster_name=test_cluster" -# - docker run --rm -v $(pwd):/app/ --workdir=/app/ -t wata727/tflint --error-with-issues -- cd examples/eks_test_fixture -- terraform init -- terraform fmt -check=true -- terraform validate -- cd - +- | + for example in $(find examples -maxdepth 1 -mindepth 1 -type d); do + cd $example + terraform init + terraform fmt -check=true + terraform validate + cd - + done - terraform -v -# - bundle exec kitchen test --destroy always -# deploy: -# provider: script -# script: ci/deploy.sh -# on: -# branch: master notifications: email: diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fdb17b263..96ce829647 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,15 @@ project adheres to [Semantic Versioning](http://semver.org/). # History +## [[v4.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1)] - 2019-05-07] + +### Changed + +- Added 2 new examples, also tidy up basic example (by @max-rocket-internet) +- Updates to travis, PR template (by @max-rocket-internet) +- Fix typo in data.tf (by @max-rocket-internet) +- Add missing launch template items in `aws_auth.tf` (by @max-rocket-internet) + ## [[v4.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1)] - 2019-05-07] ### Changed diff --git a/README.md b/README.md index 33cc7e7146..76280149b2 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aw ## Usage example -A full example leveraging other community modules is contained in the [examples/eks_test_fixture directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_test_fixture). Here's the gist of using it via the Terraform registry: +A full example leveraging other community modules is contained in the [examples/basic directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). Here's the gist of using it via the Terraform registry: ```hcl module "my-cluster" { @@ -89,11 +89,6 @@ Report issues/questions/feature requests on in the [issues](https://github.com/t Full contributing [guidelines are covered here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/CONTRIBUTING.md). -## IAM Permissions - -Testing and using this repo requires a minimum set of IAM permissions. Test permissions -are listed in the [eks_test_fixture README](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_test_fixture/README.md). - ## Change log The [changelog](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/CHANGELOG.md) captures all important release notes. @@ -133,11 +128,11 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | manage\_aws\_auth | Whether to apply the aws-auth configmap file. | string | `"true"` | no | | manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | string | `"true"` | no | | manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | string | `"true"` | no | -| map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/eks_test_fixture/variables.tf for example format. | list | `[]` | no | +| map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list | `[]` | no | | map\_accounts\_count | The count of accounts in the map_accounts list. | string | `"0"` | no | -| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/eks_test_fixture/variables.tf for example format. | list | `[]` | no | +| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list | `[]` | no | | map\_roles\_count | The count of roles in the map_roles list. | string | `"0"` | no | -| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/eks_test_fixture/variables.tf for example format. | list | `[]` | no | +| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list | `[]` | no | | map\_users\_count | The count of roles in the map_users list. | string | `"0"` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `""` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | list | n/a | yes | diff --git a/aws_auth.tf b/aws_auth.tf index a9fc899f26..abb6e0207c 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -33,6 +33,15 @@ EOS data "aws_caller_identity" "current" {} +data "template_file" "launch_template_mixed_worker_role_arns" { + count = "${var.worker_group_launch_template_mixed_count}" + template = "${file("${path.module}/templates/worker-role.tpl")}" + + vars { + worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element(coalescelist(aws_iam_instance_profile.workers_launch_template_mixed.*.role, data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.role_name), count.index)}" + } +} + data "template_file" "launch_template_worker_role_arns" { count = "${var.worker_group_launch_template_count}" template = "${file("${path.module}/templates/worker-role.tpl")}" @@ -55,7 +64,7 @@ data "template_file" "config_map_aws_auth" { template = "${file("${path.module}/templates/config-map-aws-auth.yaml.tpl")}" vars { - worker_role_arn = "${join("", distinct(concat(data.template_file.launch_template_worker_role_arns.*.rendered, data.template_file.worker_role_arns.*.rendered)))}" + worker_role_arn = "${join("", distinct(concat(data.template_file.launch_template_worker_role_arns.*.rendered, data.template_file.worker_role_arns.*.rendered, data.template_file.launch_template_mixed_worker_role_arns.*.rendered)))}" map_users = "${join("", data.template_file.map_users.*.rendered)}" map_roles = "${join("", data.template_file.map_roles.*.rendered)}" map_accounts = "${join("", data.template_file.map_accounts.*.rendered)}" diff --git a/data.tf b/data.tf index e1ec701d2b..a103b8cbc5 100644 --- a/data.tf +++ b/data.tf @@ -109,10 +109,10 @@ data "template_file" "workers_launch_template_mixed" { cluster_name = "${aws_eks_cluster.this.name}" endpoint = "${aws_eks_cluster.this.endpoint}" cluster_auth_base64 = "${aws_eks_cluster.this.certificate_authority.0.data}" - pre_userdata = "${lookup(var.worker_groups_launch_template[count.index], "pre_userdata", local.workers_group_defaults["pre_userdata"])}" - additional_userdata = "${lookup(var.worker_groups_launch_template[count.index], "additional_userdata", local.workers_group_defaults["additional_userdata"])}" - bootstrap_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "bootstrap_extra_args", local.workers_group_defaults["bootstrap_extra_args"])}" - kubelet_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "kubelet_extra_args", local.workers_group_defaults["kubelet_extra_args"])}" + pre_userdata = "${lookup(var.worker_groups_launch_template_mixed[count.index], "pre_userdata", local.workers_group_defaults["pre_userdata"])}" + additional_userdata = "${lookup(var.worker_groups_launch_template_mixed[count.index], "additional_userdata", local.workers_group_defaults["additional_userdata"])}" + bootstrap_extra_args = "${lookup(var.worker_groups_launch_template_mixed[count.index], "bootstrap_extra_args", local.workers_group_defaults["bootstrap_extra_args"])}" + kubelet_extra_args = "${lookup(var.worker_groups_launch_template_mixed[count.index], "kubelet_extra_args", local.workers_group_defaults["kubelet_extra_args"])}" } } diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000000..d1da18ab43 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,7 @@ +# Examples + +These serve a few purposes: + +1. Shows developers how to use the module in a straightforward way as integrated with other terraform community supported modules. +2. Serves as the test infrastructure for CI on the project. +3. Provides a simple way to play with the Kubernetes cluster you create. diff --git a/examples/eks_test_fixture/main.tf b/examples/basic/main.tf similarity index 50% rename from examples/eks_test_fixture/main.tf rename to examples/basic/main.tf index 058c1e3346..6c2fd6af40 100644 --- a/examples/eks_test_fixture/main.tf +++ b/examples/basic/main.tf @@ -15,71 +15,6 @@ data "aws_availability_zones" "available" {} locals { cluster_name = "test-eks-${random_string.suffix.result}" - - # the commented out worker group list below shows an example of how to define - # multiple worker groups of differing configurations - # worker_groups = [ - # { - # asg_desired_capacity = 2 - # asg_max_size = 10 - # asg_min_size = 2 - # instance_type = "m4.xlarge" - # name = "worker_group_a" - # additional_userdata = "echo foo bar" - # subnets = "${join(",", module.vpc.private_subnets)}" - # }, - # { - # asg_desired_capacity = 1 - # asg_max_size = 5 - # asg_min_size = 1 - # instance_type = "m4.2xlarge" - # name = "worker_group_b" - # additional_userdata = "echo foo bar" - # subnets = "${join(",", module.vpc.private_subnets)}" - # }, - # ] - - - # the commented out worker group tags below shows an example of how to define - # custom tags for the worker groups ASG - # worker_group_tags = { - # worker_group_a = [ - # { - # key = "k8s.io/cluster-autoscaler/node-template/taint/nvidia.com/gpu" - # value = "gpu:NoSchedule" - # propagate_at_launch = true - # }, - # ], - # worker_group_b = [ - # { - # key = "k8s.io/cluster-autoscaler/node-template/taint/nvidia.com/gpu" - # value = "gpu:NoSchedule" - # propagate_at_launch = true - # }, - # ], - # } - - worker_groups = [ - { - instance_type = "t2.small" - additional_userdata = "echo foo bar" - asg_desired_capacity = 2 - }, - ] - worker_groups_launch_template = [ - { - instance_type = "t2.small" - additional_userdata = "echo foo bar" - additional_security_group_ids = "${aws_security_group.worker_group_mgmt_one.id},${aws_security_group.worker_group_mgmt_two.id}" - asg_desired_capacity = 2 - }, - ] - tags = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - Workspace = "${terraform.workspace}" - } } resource "random_string" "suffix" { @@ -89,7 +24,6 @@ resource "random_string" "suffix" { resource "aws_security_group" "worker_group_mgmt_one" { name_prefix = "worker_group_mgmt_one" - description = "SG to be applied to all *nix machines" vpc_id = "${module.vpc.vpc_id}" ingress { @@ -140,24 +74,57 @@ module "vpc" { version = "1.60.0" name = "test-vpc" cidr = "10.0.0.0/16" - azs = ["${data.aws_availability_zones.available.names[0]}", "${data.aws_availability_zones.available.names[1]}", "${data.aws_availability_zones.available.names[2]}"] + azs = ["${data.aws_availability_zones.available.names}"] private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] enable_nat_gateway = true single_nat_gateway = true - tags = "${merge(local.tags, map("kubernetes.io/cluster/${local.cluster_name}", "shared"))}" + + tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/internal-elb" = "true" + } } module "eks" { - source = "../.." - cluster_name = "${local.cluster_name}" - subnets = ["${module.vpc.private_subnets}"] - tags = "${local.tags}" - vpc_id = "${module.vpc.vpc_id}" - worker_groups = "${local.worker_groups}" - worker_groups_launch_template = "${local.worker_groups_launch_template}" - worker_group_count = 1 - worker_group_launch_template_count = 1 + source = "../.." + cluster_name = "${local.cluster_name}" + subnets = ["${module.vpc.private_subnets}"] + + tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + vpc_id = "${module.vpc.vpc_id}" + worker_group_count = 2 + + worker_groups = [ + { + name = "worker-group-1" + instance_type = "t2.small" + additional_userdata = "echo foo bar" + asg_desired_capacity = 2 + additional_security_group_ids = "${aws_security_group.worker_group_mgmt_one.id}" + }, + { + name = "worker-group-2" + instance_type = "t2.medium" + additional_userdata = "echo foo bar" + additional_security_group_ids = "${aws_security_group.worker_group_mgmt_two.id}" + asg_desired_capacity = 1 + }, + ] + worker_additional_security_group_ids = ["${aws_security_group.all_worker_mgmt.id}"] map_roles = "${var.map_roles}" map_roles_count = "${var.map_roles_count}" diff --git a/examples/eks_test_fixture/outputs.tf b/examples/basic/outputs.tf similarity index 88% rename from examples/eks_test_fixture/outputs.tf rename to examples/basic/outputs.tf index 0422d74215..20ccf2935f 100644 --- a/examples/eks_test_fixture/outputs.tf +++ b/examples/basic/outputs.tf @@ -14,7 +14,7 @@ output "kubectl_config" { } output "config_map_aws_auth" { - description = "" + description = "A kubernetes configuration to authenticate to this EKS cluster." value = "${module.eks.config_map_aws_auth}" } diff --git a/examples/eks_test_fixture/variables.tf b/examples/basic/variables.tf similarity index 100% rename from examples/eks_test_fixture/variables.tf rename to examples/basic/variables.tf diff --git a/examples/eks_test_fixture/README.md b/examples/eks_test_fixture/README.md deleted file mode 100644 index a29ccef9c3..0000000000 --- a/examples/eks_test_fixture/README.md +++ /dev/null @@ -1,117 +0,0 @@ -# eks_test_fixture example - -This set of templates serves a few purposes. It: - -1. shows developers how to use the module in a straightforward way as integrated with other terraform community supported modules. -2. serves as the test infrastructure for CI on the project. -3. provides a simple way to play with the Kubernetes cluster you create. - -## IAM Permissions - -The following IAM policy is the minimum needed to execute the module from the test suite. - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "autoscaling:AttachInstances", - "autoscaling:CreateAutoScalingGroup", - "autoscaling:CreateLaunchConfiguration", - "autoscaling:CreateOrUpdateTags", - "autoscaling:DeleteAutoScalingGroup", - "autoscaling:DeleteLaunchConfiguration", - "autoscaling:DeleteTags", - "autoscaling:Describe*", - "autoscaling:DetachInstances", - "autoscaling:SetDesiredCapacity", - "autoscaling:UpdateAutoScalingGroup", - "ec2:AllocateAddress", - "ec2:AssignPrivateIpAddresses", - "ec2:Associate*", - "ec2:AttachInternetGateway", - "ec2:AttachNetworkInterface", - "ec2:AuthorizeSecurityGroupEgress", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateDefaultSubnet", - "ec2:CreateDhcpOptions", - "ec2:CreateEgressOnlyInternetGateway", - "ec2:CreateInternetGateway", - "ec2:CreateNatGateway", - "ec2:CreateNetworkInterface", - "ec2:CreateRoute", - "ec2:CreateRouteTable", - "ec2:CreateSecurityGroup", - "ec2:CreateSubnet", - "ec2:CreateTags", - "ec2:CreateVolume", - "ec2:CreateVpc", - "ec2:DeleteDhcpOptions", - "ec2:DeleteEgressOnlyInternetGateway", - "ec2:DeleteInternetGateway", - "ec2:DeleteNatGateway", - "ec2:DeleteNetworkInterface", - "ec2:DeleteRoute", - "ec2:DeleteRouteTable", - "ec2:DeleteSecurityGroup", - "ec2:DeleteSubnet", - "ec2:DeleteTags", - "ec2:DeleteVolume", - "ec2:DeleteVpc", - "ec2:DeleteVpnGateway", - "ec2:Describe*", - "ec2:DetachInternetGateway", - "ec2:DetachNetworkInterface", - "ec2:DetachVolume", - "ec2:Disassociate*", - "ec2:ModifySubnetAttribute", - "ec2:ModifyVpcAttribute", - "ec2:ModifyVpcEndpoint", - "ec2:ReleaseAddress", - "ec2:RevokeSecurityGroupEgress", - "ec2:RevokeSecurityGroupIngress", - "ec2:UpdateSecurityGroupRuleDescriptionsEgress", - "ec2:UpdateSecurityGroupRuleDescriptionsIngress", - "ec2:CreateLaunchTemplate", - "ec2:CreateLaunchTemplateVersion", - "ec2:DeleteLaunchTemplate", - "ec2:DeleteLaunchTemplateVersions", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeLaunchTemplateVersions", - "ec2:GetLaunchTemplateData", - "ec2:ModifyLaunchTemplate", - "eks:CreateCluster", - "eks:DeleteCluster", - "eks:DescribeCluster", - "eks:ListClusters", - "iam:AddRoleToInstanceProfile", - "iam:AttachRolePolicy", - "iam:CreateInstanceProfile", - "iam:CreatePolicy", - "iam:CreatePolicyVersion", - "iam:CreateRole", - "iam:DeleteInstanceProfile", - "iam:DeletePolicy", - "iam:DeleteRole", - "iam:DeleteRolePolicy", - "iam:DeleteServiceLinkedRole", - "iam:DetachRolePolicy", - "iam:GetInstanceProfile", - "iam:GetPolicy", - "iam:GetPolicyVersion", - "iam:GetRole", - "iam:GetRolePolicy", - "iam:List*", - "iam:PassRole", - "iam:PutRolePolicy", - "iam:RemoveRoleFromInstanceProfile", - "iam:UpdateAssumeRolePolicy" - ], - "Resource": "*" - } - ] -} -``` diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf new file mode 100644 index 0000000000..079ab23986 --- /dev/null +++ b/examples/launch_templates/main.tf @@ -0,0 +1,60 @@ +terraform { + required_version = ">= 0.11.8" +} + +provider "aws" { + version = ">= 2.6.0" + region = "${var.region}" +} + +provider "random" { + version = "= 1.3.1" +} + +data "aws_availability_zones" "available" {} + +locals { + cluster_name = "test-eks-lt-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "1.60.0" + name = "test-vpc-lt" + cidr = "10.0.0.0/16" + azs = ["${data.aws_availability_zones.available.names}"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + + tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } +} + +module "eks" { + source = "../.." + cluster_name = "${local.cluster_name}" + subnets = ["${module.vpc.public_subnets}"] + vpc_id = "${module.vpc.vpc_id}" + worker_group_count = 0 + worker_group_launch_template_count = 2 + + worker_groups_launch_template = [ + { + name = "worker-group-1" + instance_type = "t2.small" + asg_desired_capacity = 2 + public_ip = true + }, + { + name = "worker-group-2" + instance_type = "t2.medium" + asg_desired_capacity = 1 + public_ip = true + }, + ] +} diff --git a/examples/launch_templates/outputs.tf b/examples/launch_templates/outputs.tf new file mode 100644 index 0000000000..20ccf2935f --- /dev/null +++ b/examples/launch_templates/outputs.tf @@ -0,0 +1,24 @@ +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = "${module.eks.cluster_endpoint}" +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = "${module.eks.cluster_security_group_id}" +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = "${module.eks.kubeconfig}" +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = "${module.eks.config_map_aws_auth}" +} + +output "region" { + description = "AWS region." + value = "${var.region}" +} diff --git a/examples/launch_templates/variables.tf b/examples/launch_templates/variables.tf new file mode 100644 index 0000000000..81b8dbe73e --- /dev/null +++ b/examples/launch_templates/variables.tf @@ -0,0 +1,3 @@ +variable "region" { + default = "us-west-2" +} diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf new file mode 100644 index 0000000000..00ea6339b3 --- /dev/null +++ b/examples/spot_instances/main.tf @@ -0,0 +1,60 @@ +terraform { + required_version = ">= 0.11.8" +} + +provider "aws" { + version = ">= 2.6.0" + region = "${var.region}" +} + +provider "random" { + version = "= 1.3.1" +} + +data "aws_availability_zones" "available" {} + +locals { + cluster_name = "test-eks-spot-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "1.60.0" + name = "test-vpc-spot" + cidr = "10.0.0.0/16" + azs = ["${data.aws_availability_zones.available.names}"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + + tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } +} + +module "eks" { + source = "../.." + cluster_name = "${local.cluster_name}" + subnets = ["${module.vpc.public_subnets}"] + vpc_id = "${module.vpc.vpc_id}" + worker_group_count = 0 + worker_group_launch_template_mixed_count = 1 + + worker_groups_launch_template_mixed = [ + { + name = "spot-1" + override_instance_type_1 = "m5.large" + override_instance_type_2 = "c5.large" + override_instance_type_3 = "t3.large" + override_instance_type_4 = "r5.large" + spot_instance_pools = 4 + asg_max_size = 5 + asg_desired_capacity = 5 + kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" + public_ip = true + }, + ] +} diff --git a/examples/spot_instances/outputs.tf b/examples/spot_instances/outputs.tf new file mode 100644 index 0000000000..20ccf2935f --- /dev/null +++ b/examples/spot_instances/outputs.tf @@ -0,0 +1,24 @@ +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = "${module.eks.cluster_endpoint}" +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = "${module.eks.cluster_security_group_id}" +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = "${module.eks.kubeconfig}" +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = "${module.eks.config_map_aws_auth}" +} + +output "region" { + description = "AWS region." + value = "${var.region}" +} diff --git a/examples/spot_instances/variables.tf b/examples/spot_instances/variables.tf new file mode 100644 index 0000000000..81b8dbe73e --- /dev/null +++ b/examples/spot_instances/variables.tf @@ -0,0 +1,3 @@ +variable "region" { + default = "us-west-2" +} diff --git a/variables.tf b/variables.tf index fb769c4180..e43272367f 100644 --- a/variables.tf +++ b/variables.tf @@ -39,7 +39,7 @@ variable "write_aws_auth_config" { } variable "map_accounts" { - description = "Additional AWS account numbers to add to the aws-auth configmap. See examples/eks_test_fixture/variables.tf for example format." + description = "Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format." type = "list" default = [] } @@ -51,7 +51,7 @@ variable "map_accounts_count" { } variable "map_roles" { - description = "Additional IAM roles to add to the aws-auth configmap. See examples/eks_test_fixture/variables.tf for example format." + description = "Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format." type = "list" default = [] } @@ -63,7 +63,7 @@ variable "map_roles_count" { } variable "map_users" { - description = "Additional IAM users to add to the aws-auth configmap. See examples/eks_test_fixture/variables.tf for example format." + description = "Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format." type = "list" default = [] } diff --git a/version b/version index 82f24fdfff..bda368d509 100644 --- a/version +++ b/version @@ -1 +1 @@ -v4.0.1 +v4.0.2 From 763a3d5baa59f05d90e7390cdfa36e103eb38d23 Mon Sep 17 00:00:00 2001 From: Thuan Duong Date: Mon, 13 May 2019 14:55:54 +0700 Subject: [PATCH 013/962] Fix typos (#379) --- docs/spot-instances.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/spot-instances.md b/docs/spot-instances.md index b476151395..70987737ba 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -79,7 +79,7 @@ Launch Template support is a recent addition to both AWS and this module. It mig worker_group_launch_template_mixed_count = 1 - worker_group_launch_template_mixed = [ + worker_groups_launch_template_mixed = [ { name = "spot-1" override_instance_type_1 = "m5.large" From a9753e8303cb2a0827a432cde96ddd0aee7aa931 Mon Sep 17 00:00:00 2001 From: jnozo <50669295+jnozo@users.noreply.github.com> Date: Wed, 15 May 2019 08:29:25 -0700 Subject: [PATCH 014/962] Fix IAM instance profile toggle for mixed launch templates (#381) --- CHANGELOG.md | 2 +- workers_launch_template_mixed.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96ce829647..93fd995c3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed -- Write your awesome change here (by @you) +- Fix toggle for IAM instance profile creation for mixed launch templates (by @jnozo) # History diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf index 3222f6125f..d1f53a2644 100644 --- a/workers_launch_template_mixed.tf +++ b/workers_launch_template_mixed.tf @@ -115,7 +115,7 @@ resource "aws_launch_template" "workers_launch_template_mixed" { } resource "aws_iam_instance_profile" "workers_launch_template_mixed" { - count = "${var.worker_group_launch_template_mixed_count}" + count = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_mixed_count : 0}" name_prefix = "${aws_eks_cluster.this.name}" role = "${lookup(var.worker_groups_launch_template_mixed[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" path = "${var.iam_path}" From feb8810af24547e4524b013714d146ef3facbc8d Mon Sep 17 00:00:00 2001 From: Gauthier Date: Wed, 29 May 2019 16:04:23 +0200 Subject: [PATCH 015/962] Update EBS optimized instances type (#384) * Update EBS optimized instances type * Update changelog * Pin terraform version < 0.12 --- .travis.yml | 2 +- CHANGELOG.md | 2 +- local.tf | 29 +++++++++++++++++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7954f76975..2214281ed4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ install: - bundle install before_script: -- export TERRAFORM_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version') +- export TERRAFORM_VERSION=0.11.14 - curl --silent --output terraform.zip "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - unzip terraform.zip ; rm -f terraform.zip; chmod +x terraform - mkdir -p ${HOME}/bin ; export PATH=${PATH}:${HOME}/bin; mv terraform ${HOME}/bin/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 93fd995c3f..f619b4cdee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added -- Write your awesome addition here (by @you) +- Update EBS optimized instances type (by @gloutsch) ### Changed diff --git a/local.tf b/local.tf index 36ec38fc0a..8246dc75bb 100644 --- a/local.tf +++ b/local.tf @@ -142,12 +142,18 @@ locals { "m4.4xlarge" = true "m4.10xlarge" = true "m4.16xlarge" = true + "m5a.large" = true + "m5a.xlarge" = true + "m5a.2xlarge" = true + "m5a.4xlarge" = true + "m5a.12xlarge" = true + "m5a.24xlarge" = true "m5.large" = true "m5.xlarge" = true "m5.2xlarge" = true "m5.4xlarge" = true - "m5.9xlarge" = true - "m5.18xlarge" = true + "m5.12xlarge" = true + "m5.24xlarge" = true "m5d.large" = true "m5d.xlarge" = true "m5d.2xlarge" = true @@ -171,6 +177,18 @@ locals { "r4.4xlarge" = true "r4.8xlarge" = true "r4.16xlarge" = true + "r5a.large" = true + "r5a.xlarge" = true + "r5a.2xlarge" = true + "r5a.4xlarge" = true + "r5a.12xlarge" = true + "r5a.24xlarge" = true + "r5.large" = true + "r5.xlarge" = true + "r5.2xlarge" = true + "r5.4xlarge" = true + "r5.12xlarge" = true + "r5.24xlarge" = true "t1.micro" = false "t2.nano" = false "t2.micro" = false @@ -179,6 +197,13 @@ locals { "t2.large" = false "t2.xlarge" = false "t2.2xlarge" = false + "t3a.nano" = true + "t3a.micro" = true + "t3a.small" = true + "t3a.medium" = true + "t3a.large" = true + "t3a.xlarge" = true + "t3a.2xlarge" = true "t3.nano" = true "t3.micro" = true "t3.small" = true From 8c61f58a0118114320baf4d6c8c13e78f8cdd523 Mon Sep 17 00:00:00 2001 From: Feifei Jia Date: Wed, 29 May 2019 22:06:24 +0800 Subject: [PATCH 016/962] add additional policies for control plane log setup. (#377) * add additional policies for control plane log setup. * Fix typos (#379) * Fix IAM instance profile toggle for mixed launch templates (#381) * add additional policies for control plane log setup. * add two actions. --- examples/eks_test_fixture/README.md | 121 ++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 examples/eks_test_fixture/README.md diff --git a/examples/eks_test_fixture/README.md b/examples/eks_test_fixture/README.md new file mode 100644 index 0000000000..20c166a270 --- /dev/null +++ b/examples/eks_test_fixture/README.md @@ -0,0 +1,121 @@ +# eks_test_fixture example + +This set of templates serves a few purposes. It: + +1. shows developers how to use the module in a straightforward way as integrated with other terraform community supported modules. +2. serves as the test infrastructure for CI on the project. +3. provides a simple way to play with the Kubernetes cluster you create. + +## IAM Permissions + +The following IAM policy is the minimum needed to execute the module from the test suite. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "autoscaling:AttachInstances", + "autoscaling:CreateAutoScalingGroup", + "autoscaling:CreateLaunchConfiguration", + "autoscaling:CreateOrUpdateTags", + "autoscaling:DeleteAutoScalingGroup", + "autoscaling:DeleteLaunchConfiguration", + "autoscaling:DeleteTags", + "autoscaling:Describe*", + "autoscaling:DetachInstances", + "autoscaling:SetDesiredCapacity", + "autoscaling:UpdateAutoScalingGroup", + "autoscaling:SuspendProcesses", + "ec2:AllocateAddress", + "ec2:AssignPrivateIpAddresses", + "ec2:Associate*", + "ec2:AttachInternetGateway", + "ec2:AttachNetworkInterface", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateDefaultSubnet", + "ec2:CreateDhcpOptions", + "ec2:CreateEgressOnlyInternetGateway", + "ec2:CreateInternetGateway", + "ec2:CreateNatGateway", + "ec2:CreateNetworkInterface", + "ec2:CreateRoute", + "ec2:CreateRouteTable", + "ec2:CreateSecurityGroup", + "ec2:CreateSubnet", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:CreateVpc", + "ec2:DeleteDhcpOptions", + "ec2:DeleteEgressOnlyInternetGateway", + "ec2:DeleteInternetGateway", + "ec2:DeleteNatGateway", + "ec2:DeleteNetworkInterface", + "ec2:DeleteRoute", + "ec2:DeleteRouteTable", + "ec2:DeleteSecurityGroup", + "ec2:DeleteSubnet", + "ec2:DeleteTags", + "ec2:DeleteVolume", + "ec2:DeleteVpc", + "ec2:DeleteVpnGateway", + "ec2:Describe*", + "ec2:DetachInternetGateway", + "ec2:DetachNetworkInterface", + "ec2:DetachVolume", + "ec2:Disassociate*", + "ec2:ModifySubnetAttribute", + "ec2:ModifyVpcAttribute", + "ec2:ModifyVpcEndpoint", + "ec2:ReleaseAddress", + "ec2:RevokeSecurityGroupEgress", + "ec2:RevokeSecurityGroupIngress", + "ec2:UpdateSecurityGroupRuleDescriptionsEgress", + "ec2:UpdateSecurityGroupRuleDescriptionsIngress", + "ec2:CreateLaunchTemplate", + "ec2:CreateLaunchTemplateVersion", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:GetLaunchTemplateData", + "ec2:ModifyLaunchTemplate", + "ec2:RunInstances", + "eks:CreateCluster", + "eks:DeleteCluster", + "eks:DescribeCluster", + "eks:ListClusters", + "eks:UpdateClusterConfig", + "eks:DescribeUpdate", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreatePolicyVersion", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeleteRole", + "iam:DeleteRolePolicy", + "iam:DeleteServiceLinkedRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:List*", + "iam:PassRole", + "iam:PutRolePolicy", + "iam:RemoveRoleFromInstanceProfile", + "iam:UpdateAssumeRolePolicy" + ], + "Resource": "*" + } + ] +} +``` From 758fdab1ee770e82af805f801eb357c3ba28e9f9 Mon Sep 17 00:00:00 2001 From: Mitch Anderson Date: Wed, 12 Jun 2019 10:11:24 -0600 Subject: [PATCH 017/962] Termination Policy Option to worker ASG (#393) * Adding Worker ASG Option for termination_policies, default set to Default * Adding setting to launch template and mixed * Updated a few more things per PR notes * Change termination_policy default from "Default" to "" --- CHANGELOG.md | 1 + local.tf | 1 + workers.tf | 7 ++++--- workers_launch_template.tf | 7 ++++--- workers_launch_template_mixed.tf | 7 ++++--- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f619b4cdee..b1693cb769 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added +- Added Termination Policy Option to worker ASGs (by @undeadops) - Update EBS optimized instances type (by @gloutsch) ### Changed diff --git a/local.tf b/local.tf index 8246dc75bb..934c497e23 100644 --- a/local.tf +++ b/local.tf @@ -44,6 +44,7 @@ locals { enabled_metrics = "" # A comma delimited list of metrics to be collected i.e. GroupMinSize,GroupMaxSize,GroupDesiredCapacity placement_group = "" # The name of the placement group into which to launch the instances, if any. service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS + termination_policies = "" # A comma delimited list of policies to decide how the instances in the auto scale group should be terminated. # Settings for launch templates root_block_device_name = "${data.aws_ami.eks_worker.root_device_name}" # Root device name for workers. If non is provided, will assume default AMI was used. diff --git a/workers.tf b/workers.tf index ce20b1baaf..d98d7a00e1 100644 --- a/workers.tf +++ b/workers.tf @@ -15,12 +15,13 @@ resource "aws_autoscaling_group" "workers" { suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"] enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"] placement_group = "${lookup(var.worker_groups[count.index], "placement_group", local.workers_group_defaults["placement_group"])}" + termination_policies = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "termination_policies", ""), local.workers_group_defaults["termination_policies"])))}"] tags = ["${concat( list( map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true), map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true), - map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled" }", "value", "true", "propagate_at_launch", false), + map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled"}", "value", "true", "propagate_at_launch", false), map("key", "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}", "value", "", "propagate_at_launch", false), map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_groups[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}Gi", "propagate_at_launch", false) ), @@ -38,7 +39,7 @@ resource "aws_launch_configuration" "workers" { count = "${var.worker_group_count}" name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}" associate_public_ip_address = "${lookup(var.worker_groups[count.index], "public_ip", local.workers_group_defaults["public_ip"])}" - security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",",lookup(var.worker_groups[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"] + security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",", lookup(var.worker_groups[count.index], "additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"] iam_instance_profile = "${element(coalescelist(aws_iam_instance_profile.workers.*.id, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.name), count.index)}" image_id = "${lookup(var.worker_groups[count.index], "ami_id", local.workers_group_defaults["ami_id"])}" instance_type = "${lookup(var.worker_groups[count.index], "instance_type", local.workers_group_defaults["instance_type"])}" @@ -137,7 +138,7 @@ resource "aws_iam_role" "workers" { resource "aws_iam_instance_profile" "workers" { count = "${var.manage_worker_iam_resources ? var.worker_group_count : 0}" name_prefix = "${aws_eks_cluster.this.name}" - role = "${lookup(var.worker_groups[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" + role = "${lookup(var.worker_groups[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" path = "${var.iam_path}" } diff --git a/workers_launch_template.tf b/workers_launch_template.tf index efdafb1799..61d04422ed 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -14,6 +14,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"] enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"] placement_group = "${lookup(var.worker_groups_launch_template[count.index], "placement_group", local.workers_group_defaults["placement_group"])}" + termination_policies = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "termination_policies", ""), local.workers_group_defaults["termination_policies"])))}"] launch_template { id = "${element(aws_launch_template.workers_launch_template.*.id, count.index)}" @@ -24,7 +25,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { list( map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true), map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true), - map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups_launch_template[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled" }", "value", "true", "propagate_at_launch", false), + map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups_launch_template[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled"}", "value", "true", "propagate_at_launch", false), map("key", "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}", "value", "", "propagate_at_launch", false), map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_groups_launch_template[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}Gi", "propagate_at_launch", false) ), @@ -49,7 +50,7 @@ resource "aws_launch_template" "workers_launch_template" { security_groups = [ "${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", - "${compact(split(",",lookup(var.worker_groups_launch_template[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}", + "${compact(split(",", lookup(var.worker_groups_launch_template[count.index], "additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}", ] } @@ -95,6 +96,6 @@ resource "aws_launch_template" "workers_launch_template" { resource "aws_iam_instance_profile" "workers_launch_template" { count = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_count : 0}" name_prefix = "${aws_eks_cluster.this.name}" - role = "${lookup(var.worker_groups_launch_template[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" + role = "${lookup(var.worker_groups_launch_template[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" path = "${var.iam_path}" } diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf index d1f53a2644..02d8ba2662 100644 --- a/workers_launch_template_mixed.tf +++ b/workers_launch_template_mixed.tf @@ -14,6 +14,7 @@ resource "aws_autoscaling_group" "workers_launch_template_mixed" { suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"] enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"] placement_group = "${lookup(var.worker_groups_launch_template_mixed[count.index], "placement_group", local.workers_group_defaults["placement_group"])}" + termination_policies = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "termination_policies", ""), local.workers_group_defaults["termination_policies"])))}"] mixed_instances_policy { instances_distribution { @@ -53,7 +54,7 @@ resource "aws_autoscaling_group" "workers_launch_template_mixed" { list( map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true), map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true), - map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups_launch_template_mixed[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled" }", "value", "true", "propagate_at_launch", false), + map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups_launch_template_mixed[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled"}", "value", "true", "propagate_at_launch", false), map("key", "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}", "value", "", "propagate_at_launch", false), map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}Gi", "propagate_at_launch", false) ), @@ -74,7 +75,7 @@ resource "aws_launch_template" "workers_launch_template_mixed" { network_interfaces { associate_public_ip_address = "${lookup(var.worker_groups_launch_template_mixed[count.index], "public_ip", local.workers_group_defaults["public_ip"])}" delete_on_termination = "${lookup(var.worker_groups_launch_template_mixed[count.index], "eni_delete", local.workers_group_defaults["eni_delete"])}" - security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",",lookup(var.worker_groups_launch_template_mixed[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"] + security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",", lookup(var.worker_groups_launch_template_mixed[count.index], "additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"] } iam_instance_profile { @@ -117,6 +118,6 @@ resource "aws_launch_template" "workers_launch_template_mixed" { resource "aws_iam_instance_profile" "workers_launch_template_mixed" { count = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_mixed_count : 0}" name_prefix = "${aws_eks_cluster.this.name}" - role = "${lookup(var.worker_groups_launch_template_mixed[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" + role = "${lookup(var.worker_groups_launch_template_mixed[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" path = "${var.iam_path}" } From bf5dae00e18586ef3c340c103022247249bec7da Mon Sep 17 00:00:00 2001 From: Yurii Polishchuk <3329547+yuriipolishchuk@users.noreply.github.com> Date: Thu, 13 Jun 2019 11:03:34 +0300 Subject: [PATCH 018/962] Enable log retention for cloudwatch log groups (#387) * enable log retention for cloudwatch log groups * revert version bump * set default retention to 90 days --- CHANGELOG.md | 2 ++ README.md | 1 + cluster.tf | 8 ++++++++ variables.tf | 5 +++++ 4 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1693cb769..bbcdf5512e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added Termination Policy Option to worker ASGs (by @undeadops) - Update EBS optimized instances type (by @gloutsch) +- Enable log retention for cloudwatch log groups (by @yuriipolishchuk) + ### Changed diff --git a/README.md b/README.md index 76280149b2..6de9293476 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | string | `"false"` | no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | string | `"true"` | no | | cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. | string | `""` | no | +| cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days | string | `"90"` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers and provide API access to your current IP/32. | string | `""` | no | | cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.12"` | no | diff --git a/cluster.tf b/cluster.tf index dc789ee83e..9f27005181 100644 --- a/cluster.tf +++ b/cluster.tf @@ -1,3 +1,10 @@ +resource "aws_cloudwatch_log_group" "this" { + name = "/aws/eks/${var.cluster_name}/cluster" + retention_in_days = "${var.cluster_log_retention_in_days}" + + count = "${length(var.cluster_enabled_log_types) > 0 ? 1 : 0}" +} + resource "aws_eks_cluster" "this" { name = "${var.cluster_name}" enabled_cluster_log_types = "${var.cluster_enabled_log_types}" @@ -19,6 +26,7 @@ resource "aws_eks_cluster" "this" { depends_on = [ "aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy", "aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy", + "aws_cloudwatch_log_group.this", ] } diff --git a/variables.tf b/variables.tf index e43272367f..0061755906 100644 --- a/variables.tf +++ b/variables.tf @@ -4,6 +4,11 @@ variable "cluster_enabled_log_types" { type = "list" } +variable "cluster_log_retention_in_days" { + default = "90" + description = "Number of days to retain log events. Default retention - 90 days." +} + variable "cluster_name" { description = "Name of the EKS cluster. Also used as a prefix in names of related resources." } From 3f0601551fab8fd8b932e57fc9b6337382805cc9 Mon Sep 17 00:00:00 2001 From: Camilo Santana Date: Thu, 13 Jun 2019 01:36:58 -0700 Subject: [PATCH 019/962] adds tags to resource aws_iam_role.cluster (#390) * adds tagging to iam role Signed-off-by: camilo santana * emulates name_prefix Signed-off-by: camilo santana * incriment version Signed-off-by: camilo santana * tags iam role Signed-off-by: camilo santana * improve resource name Signed-off-by: camilo santana * removes comments Signed-off-by: camilo santana * reverts erroneous increment to release version Signed-off-by: camilo santana * reverts aws_iam_role.cluster name to orig value Signed-off-by: camilo santana * removes random suffix idea i was using random_id until i could figure out a datestamp or something here. `name_prefix` does a better job. Signed-off-by: camilo santana * terraform fmt on local.tf Signed-off-by: camilo santana --- CHANGELOG.md | 2 +- cluster.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbcdf5512e..37e48295b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,9 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added Termination Policy Option to worker ASGs (by @undeadops) - Update EBS optimized instances type (by @gloutsch) +- Added tagging for iam role created in `./cluster.tf` (@camilosantana) - Enable log retention for cloudwatch log groups (by @yuriipolishchuk) - ### Changed - Fix toggle for IAM instance profile creation for mixed launch templates (by @jnozo) diff --git a/cluster.tf b/cluster.tf index 9f27005181..50b883d499 100644 --- a/cluster.tf +++ b/cluster.tf @@ -67,6 +67,7 @@ resource "aws_iam_role" "cluster" { permissions_boundary = "${var.permissions_boundary}" path = "${var.iam_path}" force_detach_policies = true + tags = "${var.tags}" } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" { From da2c78b8bac6340e5d1bb75c163a1ced4d135db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=9B=B8=E8=BD=A9?= <9570112+nauxliu@users.noreply.github.com> Date: Wed, 19 Jun 2019 15:57:51 +0800 Subject: [PATCH 020/962] Upgrade to terraform 0.12 (#394) * run terraform upgrade tool * fix post upgrade TODOs * use strict typing for variables * upgrade examples, point them at VPC module tf 0.12 PR * remove unnecessary `coalesce()` calls coalesce(lookup(map, key, ""), default) -> lookup(map, key, default) * Fix autoscaling_enabled broken (#1) * always set a value for tags, fix coalescelist calls * always set a value for these tags * fix tag value * fix tag value * default element available * added default value * added a general default without this default - TF is throwing an error when running a destroy * Fix CI * Change vpc module back to `terraform-aws-modules/vpc/aws` in example * Update CHANGELOG.md * Change type of variable `cluster_log_retention_in_days` to number * Remove `xx_count` variables * Actual lists instead of strings with commas * Remove `xx_count` variable from docs * Replace element with list indexing * Change variable `worker_group_tags` to a attribute of worker_group * Fix workers_launch_template_mixed tags * Change override_instance_type_x variables to list. * Update CHANGELOG.md --- .travis.yml | 2 +- CHANGELOG.md | 7 +- README.md | 13 +- aws_auth.tf | 143 +++++++---- cluster.tf | 74 +++--- data.tf | 189 ++++++++++---- docs/spot-instances.md | 11 +- examples/basic/main.tf | 45 ++-- examples/basic/outputs.tf | 11 +- examples/basic/variables.tf | 24 +- examples/launch_templates/main.tf | 25 +- examples/launch_templates/outputs.tf | 11 +- examples/launch_templates/variables.tf | 1 + examples/spot_instances/main.tf | 30 +-- examples/spot_instances/outputs.tf | 11 +- examples/spot_instances/variables.tf | 1 + kubectl.tf | 5 +- local.tf | 136 +++++----- outputs.tf | 70 +++-- variables.tf | 131 ++++------ versions.tf | 4 + workers.tf | 333 +++++++++++++++++------- workers_launch_template.tf | 291 +++++++++++++++++---- workers_launch_template_mixed.tf | 343 ++++++++++++++++++++----- 24 files changed, 1275 insertions(+), 636 deletions(-) create mode 100644 versions.tf diff --git a/.travis.yml b/.travis.yml index 2214281ed4..c5fb533570 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ install: - bundle install before_script: -- export TERRAFORM_VERSION=0.11.14 +- export TERRAFORM_VERSION=0.12.2 - curl --silent --output terraform.zip "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - unzip terraform.zip ; rm -f terraform.zip; chmod +x terraform - mkdir -p ${HOME}/bin ; export PATH=${PATH}:${HOME}/bin; mv terraform ${HOME}/bin/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 37e48295b9..a1829f9908 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v4.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...HEAD)] - 2019-06-??] +## [[v5.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...HEAD)] - 2019-06-??] ### Added @@ -18,6 +18,11 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed +- Finally, Terraform 0.12 support, [Upgrade Guide](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/394) (by @alex-goncharov @nauxliu @timboven) +- All the xx_count variables have been removed (by @nauxliu on behalf of RightCapital) +- Use actual lists in the workers group maps instead of strings with commas (by @nauxliu on behalf of RightCapital) +- Move variable `worker_group_tags` to workers group's attribute `tags` (by @nauxliu on behalf of RightCapital) +- Change override instance_types to list (by @nauxliu on behalf of RightCapital) - Fix toggle for IAM instance profile creation for mixed launch templates (by @jnozo) # History diff --git a/README.md b/README.md index 6de9293476..3d15e2695a 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,11 @@ module "my-cluster" { { instance_type = "m4.large" asg_max_size = 5 + tags = { + key = "foo" + value = "bar" + propagate_at_launch = true + } } ] @@ -130,11 +135,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | string | `"true"` | no | | manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | string | `"true"` | no | | map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list | `[]` | no | -| map\_accounts\_count | The count of accounts in the map_accounts list. | string | `"0"` | no | | map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list | `[]` | no | -| map\_roles\_count | The count of roles in the map_roles list. | string | `"0"` | no | | map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list | `[]` | no | -| map\_users\_count | The count of roles in the map_users list. | string | `"0"` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `""` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | list | n/a | yes | | tags | A map of tags to add to all resources. | map | `{}` | no | @@ -142,17 +144,12 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list | `[]` | no | | worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | string | `"true"` | no | -| worker\_group\_count | The number of maps contained within the worker_groups list. | string | `"1"` | no | -| worker\_group\_launch\_template\_count | The number of maps contained within the worker_groups_launch_template list. | string | `"0"` | no | -| worker\_group\_launch\_template\_mixed\_count | The number of maps contained within the worker_groups_launch_template_mixed list. | string | `"0"` | no | -| worker\_group\_tags | A map defining extra tags to be applied to the worker group ASG. | map | `{ "default": [] }` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | | worker\_groups\_launch\_template\_mixed | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | | worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster. | string | `""` | no | | worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | string | `"1025"` | no | | workers\_additional\_policies | Additional policies to be added to workers | list | `[]` | no | -| workers\_additional\_policies\_count | | string | `"0"` | no | | workers\_group\_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | map | `{}` | no | | write\_aws\_auth\_config | Whether to write the aws-auth configmap file. | string | `"true"` | no | | write\_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | string | `"true"` | no | diff --git a/aws_auth.tf b/aws_auth.tf index abb6e0207c..c5cb532509 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -1,103 +1,140 @@ resource "local_file" "config_map_aws_auth" { - count = "${var.write_aws_auth_config ? 1 : 0}" - content = "${data.template_file.config_map_aws_auth.rendered}" + count = var.write_aws_auth_config ? 1 : 0 + content = data.template_file.config_map_aws_auth.rendered filename = "${var.config_output_path}config-map-aws-auth_${var.cluster_name}.yaml" } resource "null_resource" "update_config_map_aws_auth" { - count = "${var.manage_aws_auth ? 1 : 0}" - depends_on = ["aws_eks_cluster.this"] + count = var.manage_aws_auth ? 1 : 0 + depends_on = [aws_eks_cluster.this] provisioner "local-exec" { - working_dir = "${path.module}" + working_dir = path.module command = < kube_config.yaml & \ -echo "${null_resource.update_config_map_aws_auth.triggers.config_map_rendered}" > aws_auth_configmap.yaml & \ +echo "${null_resource.update_config_map_aws_auth[0].triggers.kube_config_map_rendered}" > kube_config.yaml & \ +echo "${null_resource.update_config_map_aws_auth[0].triggers.config_map_rendered}" > aws_auth_configmap.yaml & \ kubectl apply -f aws_auth_configmap.yaml --kubeconfig kube_config.yaml && break || \ sleep 10; \ done; \ rm aws_auth_configmap.yaml kube_config.yaml; EOS - interpreter = ["${var.local_exec_interpreter}"] + + interpreter = var.local_exec_interpreter } - triggers { - kube_config_map_rendered = "${data.template_file.kubeconfig.rendered}" - config_map_rendered = "${data.template_file.config_map_aws_auth.rendered}" - endpoint = "${aws_eks_cluster.this.endpoint}" + triggers = { + kube_config_map_rendered = data.template_file.kubeconfig.rendered + config_map_rendered = data.template_file.config_map_aws_auth.rendered + endpoint = aws_eks_cluster.this.endpoint } } -data "aws_caller_identity" "current" {} +data "aws_caller_identity" "current" { +} data "template_file" "launch_template_mixed_worker_role_arns" { - count = "${var.worker_group_launch_template_mixed_count}" - template = "${file("${path.module}/templates/worker-role.tpl")}" - - vars { - worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element(coalescelist(aws_iam_instance_profile.workers_launch_template_mixed.*.role, data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.role_name), count.index)}" + count = local.worker_group_launch_template_mixed_count + template = file("${path.module}/templates/worker-role.tpl") + + vars = { + worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element( + coalescelist( + aws_iam_instance_profile.workers_launch_template_mixed.*.role, + data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.role_name, + ), + count.index, + )}" } } data "template_file" "launch_template_worker_role_arns" { - count = "${var.worker_group_launch_template_count}" - template = "${file("${path.module}/templates/worker-role.tpl")}" - - vars { - worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element(coalescelist(aws_iam_instance_profile.workers_launch_template.*.role, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name), count.index)}" + count = local.worker_group_launch_template_count + template = file("${path.module}/templates/worker-role.tpl") + + vars = { + worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element( + coalescelist( + aws_iam_instance_profile.workers_launch_template.*.role, + data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, + ), + count.index, + )}" } } data "template_file" "worker_role_arns" { - count = "${var.worker_group_count}" - template = "${file("${path.module}/templates/worker-role.tpl")}" - - vars { - worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element(coalescelist(aws_iam_instance_profile.workers.*.role, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name), count.index)}" + count = local.worker_group_count + template = file("${path.module}/templates/worker-role.tpl") + + vars = { + worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element( + coalescelist( + aws_iam_instance_profile.workers.*.role, + data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, + [""] + ), + count.index, + )}" } } data "template_file" "config_map_aws_auth" { - template = "${file("${path.module}/templates/config-map-aws-auth.yaml.tpl")}" - - vars { - worker_role_arn = "${join("", distinct(concat(data.template_file.launch_template_worker_role_arns.*.rendered, data.template_file.worker_role_arns.*.rendered, data.template_file.launch_template_mixed_worker_role_arns.*.rendered)))}" - map_users = "${join("", data.template_file.map_users.*.rendered)}" - map_roles = "${join("", data.template_file.map_roles.*.rendered)}" - map_accounts = "${join("", data.template_file.map_accounts.*.rendered)}" + template = file("${path.module}/templates/config-map-aws-auth.yaml.tpl") + + vars = { + worker_role_arn = join( + "", + distinct( + concat( + data.template_file.launch_template_worker_role_arns.*.rendered, + data.template_file.worker_role_arns.*.rendered, + data.template_file.launch_template_mixed_worker_role_arns.*.rendered, + ), + ), + ) + map_users = join("", data.template_file.map_users.*.rendered) + map_roles = join("", data.template_file.map_roles.*.rendered) + map_accounts = join("", data.template_file.map_accounts.*.rendered) } } data "template_file" "map_users" { - count = "${var.map_users_count}" - template = "${file("${path.module}/templates/config-map-aws-auth-map_users.yaml.tpl")}" - - vars { - user_arn = "${lookup(var.map_users[count.index], "user_arn")}" - username = "${lookup(var.map_users[count.index], "username")}" - group = "${lookup(var.map_users[count.index], "group")}" + count = length(var.map_users) + template = file( + "${path.module}/templates/config-map-aws-auth-map_users.yaml.tpl", + ) + + vars = { + user_arn = var.map_users[count.index]["user_arn"] + username = var.map_users[count.index]["username"] + group = var.map_users[count.index]["group"] } } data "template_file" "map_roles" { - count = "${var.map_roles_count}" - template = "${file("${path.module}/templates/config-map-aws-auth-map_roles.yaml.tpl")}" - - vars { - role_arn = "${lookup(var.map_roles[count.index], "role_arn")}" - username = "${lookup(var.map_roles[count.index], "username")}" - group = "${lookup(var.map_roles[count.index], "group")}" + count = length(var.map_roles) + template = file( + "${path.module}/templates/config-map-aws-auth-map_roles.yaml.tpl", + ) + + vars = { + role_arn = var.map_roles[count.index]["role_arn"] + username = var.map_roles[count.index]["username"] + group = var.map_roles[count.index]["group"] } } data "template_file" "map_accounts" { - count = "${var.map_accounts_count}" - template = "${file("${path.module}/templates/config-map-aws-auth-map_accounts.yaml.tpl")}" + count = length(var.map_accounts) + template = file( + "${path.module}/templates/config-map-aws-auth-map_accounts.yaml.tpl", + ) - vars { - account_number = "${element(var.map_accounts, count.index)}" + vars = { + account_number = var.map_accounts[count.index] } } + diff --git a/cluster.tf b/cluster.tf index 50b883d499..e39eaf5776 100644 --- a/cluster.tf +++ b/cluster.tf @@ -1,48 +1,53 @@ resource "aws_cloudwatch_log_group" "this" { name = "/aws/eks/${var.cluster_name}/cluster" - retention_in_days = "${var.cluster_log_retention_in_days}" + retention_in_days = var.cluster_log_retention_in_days - count = "${length(var.cluster_enabled_log_types) > 0 ? 1 : 0}" + count = length(var.cluster_enabled_log_types) > 0 ? 1 : 0 } resource "aws_eks_cluster" "this" { - name = "${var.cluster_name}" - enabled_cluster_log_types = "${var.cluster_enabled_log_types}" - role_arn = "${local.cluster_iam_role_arn}" - version = "${var.cluster_version}" + name = var.cluster_name + enabled_cluster_log_types = var.cluster_enabled_log_types + role_arn = local.cluster_iam_role_arn + version = var.cluster_version vpc_config { - security_group_ids = ["${local.cluster_security_group_id}"] - subnet_ids = ["${var.subnets}"] - endpoint_private_access = "${var.cluster_endpoint_private_access}" - endpoint_public_access = "${var.cluster_endpoint_public_access}" + security_group_ids = [local.cluster_security_group_id] + subnet_ids = var.subnets + endpoint_private_access = var.cluster_endpoint_private_access + endpoint_public_access = var.cluster_endpoint_public_access } timeouts { - create = "${var.cluster_create_timeout}" - delete = "${var.cluster_delete_timeout}" + create = var.cluster_create_timeout + delete = var.cluster_delete_timeout } depends_on = [ - "aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy", - "aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy", - "aws_cloudwatch_log_group.this", + aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy, + aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy, + aws_cloudwatch_log_group.this ] } resource "aws_security_group" "cluster" { - count = "${var.cluster_create_security_group ? 1 : 0}" - name_prefix = "${var.cluster_name}" + count = var.cluster_create_security_group ? 1 : 0 + name_prefix = var.cluster_name description = "EKS cluster security group." - vpc_id = "${var.vpc_id}" - tags = "${merge(var.tags, map("Name", "${var.cluster_name}-eks_cluster_sg"))}" + vpc_id = var.vpc_id + tags = merge( + var.tags, + { + "Name" = "${var.cluster_name}-eks_cluster_sg" + }, + ) } resource "aws_security_group_rule" "cluster_egress_internet" { - count = "${var.cluster_create_security_group ? 1 : 0}" + count = var.cluster_create_security_group ? 1 : 0 description = "Allow cluster egress access to the Internet." protocol = "-1" - security_group_id = "${aws_security_group.cluster.id}" + security_group_id = aws_security_group.cluster[0].id cidr_blocks = ["0.0.0.0/0"] from_port = 0 to_port = 0 @@ -50,34 +55,35 @@ resource "aws_security_group_rule" "cluster_egress_internet" { } resource "aws_security_group_rule" "cluster_https_worker_ingress" { - count = "${var.cluster_create_security_group ? 1 : 0}" + count = var.cluster_create_security_group ? 1 : 0 description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" - security_group_id = "${aws_security_group.cluster.id}" - source_security_group_id = "${local.worker_security_group_id}" + security_group_id = aws_security_group.cluster[0].id + source_security_group_id = local.worker_security_group_id from_port = 443 to_port = 443 type = "ingress" } resource "aws_iam_role" "cluster" { - count = "${var.manage_cluster_iam_resources ? 1 : 0}" - name_prefix = "${var.cluster_name}" - assume_role_policy = "${data.aws_iam_policy_document.cluster_assume_role_policy.json}" - permissions_boundary = "${var.permissions_boundary}" - path = "${var.iam_path}" + count = var.manage_cluster_iam_resources ? 1 : 0 + name_prefix = var.cluster_name + assume_role_policy = data.aws_iam_policy_document.cluster_assume_role_policy.json + permissions_boundary = var.permissions_boundary + path = var.iam_path force_detach_policies = true - tags = "${var.tags}" + tags = var.tags } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" { - count = "${var.manage_cluster_iam_resources ? 1 : 0}" + count = var.manage_cluster_iam_resources ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" - role = "${aws_iam_role.cluster.name}" + role = aws_iam_role.cluster[0].name } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { - count = "${var.manage_cluster_iam_resources ? 1 : 0}" + count = var.manage_cluster_iam_resources ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" - role = "${aws_iam_role.cluster.name}" + role = aws_iam_role.cluster[0].name } + diff --git a/data.tf b/data.tf index a103b8cbc5..f7f7a33233 100644 --- a/data.tf +++ b/data.tf @@ -1,4 +1,5 @@ -data "aws_region" "current" {} +data "aws_region" "current" { +} data "aws_iam_policy_document" "workers_assume_role_policy" { statement { @@ -43,95 +44,169 @@ data "aws_iam_policy_document" "cluster_assume_role_policy" { } data "template_file" "kubeconfig" { - template = "${file("${path.module}/templates/kubeconfig.tpl")}" - - vars { - kubeconfig_name = "${local.kubeconfig_name}" - endpoint = "${aws_eks_cluster.this.endpoint}" - region = "${data.aws_region.current.name}" - cluster_auth_base64 = "${aws_eks_cluster.this.certificate_authority.0.data}" - aws_authenticator_command = "${var.kubeconfig_aws_authenticator_command}" - aws_authenticator_command_args = "${length(var.kubeconfig_aws_authenticator_command_args) > 0 ? " - ${join("\n - ", var.kubeconfig_aws_authenticator_command_args)}" : " - ${join("\n - ", formatlist("\"%s\"", list("token", "-i", aws_eks_cluster.this.name)))}"}" - aws_authenticator_additional_args = "${length(var.kubeconfig_aws_authenticator_additional_args) > 0 ? " - ${join("\n - ", var.kubeconfig_aws_authenticator_additional_args)}" : ""}" - aws_authenticator_env_variables = "${length(var.kubeconfig_aws_authenticator_env_variables) > 0 ? " env:\n${join("\n", data.template_file.aws_authenticator_env_variables.*.rendered)}" : ""}" + template = file("${path.module}/templates/kubeconfig.tpl") + + vars = { + kubeconfig_name = local.kubeconfig_name + endpoint = aws_eks_cluster.this.endpoint + region = data.aws_region.current.name + cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data + aws_authenticator_command = var.kubeconfig_aws_authenticator_command + aws_authenticator_command_args = length(var.kubeconfig_aws_authenticator_command_args) > 0 ? " - ${join( + "\n - ", + var.kubeconfig_aws_authenticator_command_args, + )}" : " - ${join( + "\n - ", + formatlist("\"%s\"", ["token", "-i", aws_eks_cluster.this.name]), + )}" + aws_authenticator_additional_args = length(var.kubeconfig_aws_authenticator_additional_args) > 0 ? " - ${join( + "\n - ", + var.kubeconfig_aws_authenticator_additional_args, + )}" : "" + aws_authenticator_env_variables = length(var.kubeconfig_aws_authenticator_env_variables) > 0 ? " env:\n${join( + "\n", + data.template_file.aws_authenticator_env_variables.*.rendered, + )}" : "" } } data "template_file" "aws_authenticator_env_variables" { - count = "${length(var.kubeconfig_aws_authenticator_env_variables)}" + count = length(var.kubeconfig_aws_authenticator_env_variables) template = < 10250 ? 1 : 0 : 0 description = "Allow workers Kubelets to receive communication from the cluster control plane." protocol = "tcp" - security_group_id = "${aws_security_group.workers.id}" - source_security_group_id = "${local.cluster_security_group_id}" + security_group_id = aws_security_group.workers[0].id + source_security_group_id = local.cluster_security_group_id from_port = 10250 to_port = 10250 type = "ingress" } resource "aws_security_group_rule" "workers_ingress_cluster_https" { - count = "${var.worker_create_security_group ? 1 : 0}" + count = var.worker_create_security_group ? 1 : 0 description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane." protocol = "tcp" - security_group_id = "${aws_security_group.workers.id}" - source_security_group_id = "${local.cluster_security_group_id}" + security_group_id = aws_security_group.workers[0].id + source_security_group_id = local.cluster_security_group_id from_port = 443 to_port = 443 type = "ingress" } resource "aws_iam_role" "workers" { - count = "${var.manage_worker_iam_resources ? 1 : 0}" - name_prefix = "${aws_eks_cluster.this.name}" - assume_role_policy = "${data.aws_iam_policy_document.workers_assume_role_policy.json}" - permissions_boundary = "${var.permissions_boundary}" - path = "${var.iam_path}" + count = var.manage_worker_iam_resources ? 1 : 0 + name_prefix = aws_eks_cluster.this.name + assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json + permissions_boundary = var.permissions_boundary + path = var.iam_path force_detach_policies = true } resource "aws_iam_instance_profile" "workers" { - count = "${var.manage_worker_iam_resources ? var.worker_group_count : 0}" - name_prefix = "${aws_eks_cluster.this.name}" - role = "${lookup(var.worker_groups[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" + count = var.manage_worker_iam_resources ? local.worker_group_count : 0 + name_prefix = aws_eks_cluster.this.name + role = lookup( + var.worker_groups[count.index], + "iam_role_id", + local.workers_group_defaults["iam_role_id"], + ) - path = "${var.iam_path}" + path = var.iam_path } resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { - count = "${var.manage_worker_iam_resources ? 1 : 0}" + count = var.manage_worker_iam_resources ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" - role = "${aws_iam_role.workers.name}" + role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { - count = "${var.manage_worker_iam_resources ? 1 : 0}" + count = var.manage_worker_iam_resources ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" - role = "${aws_iam_role.workers.name}" + role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" { - count = "${var.manage_worker_iam_resources ? 1 : 0}" + count = var.manage_worker_iam_resources ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" - role = "${aws_iam_role.workers.name}" + role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_additional_policies" { - count = "${var.manage_worker_iam_resources ? var.workers_additional_policies_count : 0}" - role = "${aws_iam_role.workers.name}" - policy_arn = "${var.workers_additional_policies[count.index]}" + count = var.manage_worker_iam_resources ? length(var.workers_additional_policies) : 0 + role = aws_iam_role.workers[0].name + policy_arn = var.workers_additional_policies[count.index] } resource "null_resource" "tags_as_list_of_maps" { - count = "${length(keys(var.tags))}" + count = length(keys(var.tags)) triggers = { - key = "${element(keys(var.tags), count.index)}" - value = "${element(values(var.tags), count.index)}" + key = keys(var.tags)[count.index] + value = values(var.tags)[count.index] propagate_at_launch = "true" } } resource "aws_iam_role_policy_attachment" "workers_autoscaling" { - count = "${var.manage_worker_iam_resources ? 1 : 0}" - policy_arn = "${aws_iam_policy.worker_autoscaling.arn}" - role = "${aws_iam_role.workers.name}" + count = var.manage_worker_iam_resources ? 1 : 0 + policy_arn = aws_iam_policy.worker_autoscaling[0].arn + role = aws_iam_role.workers[0].name } resource "aws_iam_policy" "worker_autoscaling" { - count = "${var.manage_worker_iam_resources ? 1 : 0}" + count = var.manage_worker_iam_resources ? 1 : 0 name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this.name}" description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this.name}" - policy = "${data.aws_iam_policy_document.worker_autoscaling.json}" - path = "${var.iam_path}" + policy = data.aws_iam_policy_document.worker_autoscaling.json + path = var.iam_path } data "aws_iam_policy_document" "worker_autoscaling" { @@ -232,3 +384,4 @@ data "aws_iam_policy_document" "worker_autoscaling" { } } } + diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 61d04422ed..931c18cea3 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -1,92 +1,264 @@ # Worker Groups using Launch Templates resource "aws_autoscaling_group" "workers_launch_template" { - count = "${var.worker_group_launch_template_count}" - name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}" - desired_capacity = "${lookup(var.worker_groups_launch_template[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}" - max_size = "${lookup(var.worker_groups_launch_template[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}" - min_size = "${lookup(var.worker_groups_launch_template[count.index], "asg_min_size", local.workers_group_defaults["asg_min_size"])}" - force_delete = "${lookup(var.worker_groups_launch_template[count.index], "asg_force_delete", local.workers_group_defaults["asg_force_delete"])}" - target_group_arns = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "target_group_arns", ""), local.workers_group_defaults["target_group_arns"])))}"] - service_linked_role_arn = "${lookup(var.worker_groups_launch_template[count.index], "service_linked_role_arn", local.workers_group_defaults["service_linked_role_arn"])}" - vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "subnets", ""), local.workers_group_defaults["subnets"]))}"] - protect_from_scale_in = "${lookup(var.worker_groups_launch_template[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}" - suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"] - enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"] - placement_group = "${lookup(var.worker_groups_launch_template[count.index], "placement_group", local.workers_group_defaults["placement_group"])}" - termination_policies = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "termination_policies", ""), local.workers_group_defaults["termination_policies"])))}"] + count = local.worker_group_launch_template_count + name_prefix = "${aws_eks_cluster.this.name}-${lookup( + var.worker_groups_launch_template[count.index], + "name", + count.index, + )}" + desired_capacity = lookup( + var.worker_groups_launch_template[count.index], + "asg_desired_capacity", + local.workers_group_defaults["asg_desired_capacity"], + ) + max_size = lookup( + var.worker_groups_launch_template[count.index], + "asg_max_size", + local.workers_group_defaults["asg_max_size"], + ) + min_size = lookup( + var.worker_groups_launch_template[count.index], + "asg_min_size", + local.workers_group_defaults["asg_min_size"], + ) + force_delete = lookup( + var.worker_groups_launch_template[count.index], + "asg_force_delete", + local.workers_group_defaults["asg_force_delete"], + ) + target_group_arns = lookup( + var.worker_groups_launch_template[count.index], + "target_group_arns", + local.workers_group_defaults["target_group_arns"] + ) + service_linked_role_arn = lookup( + var.worker_groups_launch_template[count.index], + "service_linked_role_arn", + local.workers_group_defaults["service_linked_role_arn"], + ) + vpc_zone_identifier = lookup( + var.worker_groups_launch_template[count.index], + "subnets", + local.workers_group_defaults["subnets"] + ) + protect_from_scale_in = lookup( + var.worker_groups_launch_template[count.index], + "protect_from_scale_in", + local.workers_group_defaults["protect_from_scale_in"], + ) + suspended_processes = lookup( + var.worker_groups_launch_template[count.index], + "suspended_processes", + local.workers_group_defaults["suspended_processes"] + ) + enabled_metrics = lookup( + var.worker_groups_launch_template[count.index], + "enabled_metrics", + local.workers_group_defaults["enabled_metrics"] + ) + placement_group = lookup( + var.worker_groups_launch_template[count.index], + "placement_group", + local.workers_group_defaults["placement_group"], + ) + termination_policies = lookup( + var.worker_groups_launch_template[count.index], + "termination_policies", + local.workers_group_defaults["termination_policies"] + ) launch_template { - id = "${element(aws_launch_template.workers_launch_template.*.id, count.index)}" - version = "${lookup(var.worker_groups_launch_template[count.index], "launch_template_version", local.workers_group_defaults["launch_template_version"])}" + id = aws_launch_template.workers_launch_template.*.id[count.index] + version = lookup( + var.worker_groups_launch_template[count.index], + "launch_template_version", + local.workers_group_defaults["launch_template_version"], + ) } - tags = ["${concat( - list( - map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true), - map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true), - map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups_launch_template[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled"}", "value", "true", "propagate_at_launch", false), - map("key", "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}", "value", "", "propagate_at_launch", false), - map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_groups_launch_template[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}Gi", "propagate_at_launch", false) - ), + tags = concat( + [ + { + "key" = "Name" + "value" = "${aws_eks_cluster.this.name}-${lookup( + var.worker_groups_launch_template[count.index], + "name", + count.index, + )}-eks_asg" + "propagate_at_launch" = true + }, + { + "key" = "kubernetes.io/cluster/${aws_eks_cluster.this.name}" + "value" = "owned" + "propagate_at_launch" = true + }, + { + "key" = "k8s.io/cluster-autoscaler/${lookup( + var.worker_groups_launch_template[count.index], + "autoscaling_enabled", + local.workers_group_defaults["autoscaling_enabled"], + ) ? "enabled" : "disabled"}" + "value" = "true" + "propagate_at_launch" = false + }, + { + "key" = "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}" + "value" = aws_eks_cluster.this.name + "propagate_at_launch" = false + }, + { + "key" = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" + "value" = "${lookup( + var.worker_groups_launch_template[count.index], + "root_volume_size", + local.workers_group_defaults["root_volume_size"], + )}Gi" + "propagate_at_launch" = false + }, + ], local.asg_tags, - var.worker_group_tags[contains(keys(var.worker_group_tags), "${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}") ? "${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}" : "default"]) - }"] + lookup( + var.worker_groups_launch_template[count.index], + "tags", + local.workers_group_defaults["tags"] + ) + ) lifecycle { create_before_destroy = true - ignore_changes = ["desired_capacity"] + ignore_changes = [desired_capacity] } } resource "aws_launch_template" "workers_launch_template" { - count = "${var.worker_group_launch_template_count}" - name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}" + count = local.worker_group_launch_template_count + name_prefix = "${aws_eks_cluster.this.name}-${lookup( + var.worker_groups_launch_template[count.index], + "name", + count.index, + )}" network_interfaces { - associate_public_ip_address = "${lookup(var.worker_groups_launch_template[count.index], "public_ip", local.workers_group_defaults["public_ip"])}" - delete_on_termination = "${lookup(var.worker_groups_launch_template[count.index], "eni_delete", local.workers_group_defaults["eni_delete"])}" - - security_groups = [ - "${local.worker_security_group_id}", - "${var.worker_additional_security_group_ids}", - "${compact(split(",", lookup(var.worker_groups_launch_template[count.index], "additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}", - ] + associate_public_ip_address = lookup( + var.worker_groups_launch_template[count.index], + "public_ip", + local.workers_group_defaults["public_ip"], + ) + delete_on_termination = lookup( + var.worker_groups_launch_template[count.index], + "eni_delete", + local.workers_group_defaults["eni_delete"], + ) + security_groups = flatten([ + local.worker_security_group_id, + var.worker_additional_security_group_ids, + lookup( + var.worker_groups_launch_template[count.index], + "additional_security_group_ids", + local.workers_group_defaults["additional_security_group_ids"], + ), + ]) } iam_instance_profile { - name = "${element(coalescelist(aws_iam_instance_profile.workers_launch_template.*.name, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.name), count.index)}" + name = coalescelist( + aws_iam_instance_profile.workers_launch_template.*.name, + data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.name, + )[count.index] } - image_id = "${lookup(var.worker_groups_launch_template[count.index], "ami_id", local.workers_group_defaults["ami_id"])}" - instance_type = "${lookup(var.worker_groups_launch_template[count.index], "instance_type", local.workers_group_defaults["instance_type"])}" - key_name = "${lookup(var.worker_groups_launch_template[count.index], "key_name", local.workers_group_defaults["key_name"])}" - user_data = "${base64encode(element(data.template_file.launch_template_userdata.*.rendered, count.index))}" - ebs_optimized = "${lookup(var.worker_groups_launch_template[count.index], "ebs_optimized", lookup(local.ebs_optimized, lookup(var.worker_groups_launch_template[count.index], "instance_type", local.workers_group_defaults["instance_type"]), false))}" + image_id = lookup( + var.worker_groups_launch_template[count.index], + "ami_id", + local.workers_group_defaults["ami_id"], + ) + instance_type = lookup( + var.worker_groups_launch_template[count.index], + "instance_type", + local.workers_group_defaults["instance_type"], + ) + key_name = lookup( + var.worker_groups_launch_template[count.index], + "key_name", + local.workers_group_defaults["key_name"], + ) + user_data = base64encode( + data.template_file.launch_template_userdata.*.rendered[count.index], + ) + ebs_optimized = lookup( + var.worker_groups_launch_template[count.index], + "ebs_optimized", + lookup( + local.ebs_optimized, + lookup( + var.worker_groups_launch_template[count.index], + "instance_type", + local.workers_group_defaults["instance_type"], + ), + false, + ), + ) monitoring { - enabled = "${lookup(var.worker_groups_launch_template[count.index], "enable_monitoring", local.workers_group_defaults["enable_monitoring"])}" + enabled = lookup( + var.worker_groups_launch_template[count.index], + "enable_monitoring", + local.workers_group_defaults["enable_monitoring"], + ) } placement { - tenancy = "${lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_tenancy", local.workers_group_defaults["launch_template_placement_tenancy"])}" - group_name = "${lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])}" + tenancy = lookup( + var.worker_groups_launch_template[count.index], + "launch_template_placement_tenancy", + local.workers_group_defaults["launch_template_placement_tenancy"], + ) + group_name = lookup( + var.worker_groups_launch_template[count.index], + "launch_template_placement_group", + local.workers_group_defaults["launch_template_placement_group"], + ) } block_device_mappings { - device_name = "${lookup(var.worker_groups_launch_template[count.index], "root_block_device_name", local.workers_group_defaults["root_block_device_name"])}" + device_name = lookup( + var.worker_groups_launch_template[count.index], + "root_block_device_name", + local.workers_group_defaults["root_block_device_name"], + ) ebs { - volume_size = "${lookup(var.worker_groups_launch_template[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}" - volume_type = "${lookup(var.worker_groups_launch_template[count.index], "root_volume_type", local.workers_group_defaults["root_volume_type"])}" - iops = "${lookup(var.worker_groups_launch_template[count.index], "root_iops", local.workers_group_defaults["root_iops"])}" - encrypted = "${lookup(var.worker_groups_launch_template[count.index], "root_encrypted", local.workers_group_defaults["root_encrypted"])}" - kms_key_id = "${lookup(var.worker_groups_launch_template[count.index], "root_kms_key_id", local.workers_group_defaults["root_kms_key_id"])}" + volume_size = lookup( + var.worker_groups_launch_template[count.index], + "root_volume_size", + local.workers_group_defaults["root_volume_size"], + ) + volume_type = lookup( + var.worker_groups_launch_template[count.index], + "root_volume_type", + local.workers_group_defaults["root_volume_type"], + ) + iops = lookup( + var.worker_groups_launch_template[count.index], + "root_iops", + local.workers_group_defaults["root_iops"], + ) + encrypted = lookup( + var.worker_groups_launch_template[count.index], + "root_encrypted", + local.workers_group_defaults["root_encrypted"], + ) + kms_key_id = lookup( + var.worker_groups_launch_template[count.index], + "root_kms_key_id", + local.workers_group_defaults["root_kms_key_id"], + ) delete_on_termination = true } } - tags = "${var.tags}" + tags = var.tags lifecycle { create_before_destroy = true @@ -94,8 +266,13 @@ resource "aws_launch_template" "workers_launch_template" { } resource "aws_iam_instance_profile" "workers_launch_template" { - count = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_count : 0}" - name_prefix = "${aws_eks_cluster.this.name}" - role = "${lookup(var.worker_groups_launch_template[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" - path = "${var.iam_path}" + count = var.manage_worker_iam_resources ? local.worker_group_launch_template_count : 0 + name_prefix = aws_eks_cluster.this.name + role = lookup( + var.worker_groups_launch_template[count.index], + "iam_role_id", + local.workers_group_defaults["iam_role_id"], + ) + path = var.iam_path } + diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf index 02d8ba2662..7de2c9a09e 100644 --- a/workers_launch_template_mixed.tf +++ b/workers_launch_template_mixed.tf @@ -1,123 +1,328 @@ # Worker Groups using Launch Templates with mixed instances policy resource "aws_autoscaling_group" "workers_launch_template_mixed" { - count = "${var.worker_group_launch_template_mixed_count}" - name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}" - desired_capacity = "${lookup(var.worker_groups_launch_template_mixed[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}" - max_size = "${lookup(var.worker_groups_launch_template_mixed[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}" - min_size = "${lookup(var.worker_groups_launch_template_mixed[count.index], "asg_min_size", local.workers_group_defaults["asg_min_size"])}" - force_delete = "${lookup(var.worker_groups_launch_template_mixed[count.index], "asg_force_delete", local.workers_group_defaults["asg_force_delete"])}" - target_group_arns = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "target_group_arns", ""), local.workers_group_defaults["target_group_arns"])))}"] - service_linked_role_arn = "${lookup(var.worker_groups_launch_template_mixed[count.index], "service_linked_role_arn", local.workers_group_defaults["service_linked_role_arn"])}" - vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "subnets", ""), local.workers_group_defaults["subnets"]))}"] - protect_from_scale_in = "${lookup(var.worker_groups_launch_template_mixed[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}" - suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"] - enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"] - placement_group = "${lookup(var.worker_groups_launch_template_mixed[count.index], "placement_group", local.workers_group_defaults["placement_group"])}" - termination_policies = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template_mixed[count.index], "termination_policies", ""), local.workers_group_defaults["termination_policies"])))}"] + count = local.worker_group_launch_template_mixed_count + name_prefix = "${aws_eks_cluster.this.name}-${lookup( + var.worker_groups_launch_template_mixed[count.index], + "name", + count.index, + )}" + desired_capacity = lookup( + var.worker_groups_launch_template_mixed[count.index], + "asg_desired_capacity", + local.workers_group_defaults["asg_desired_capacity"], + ) + max_size = lookup( + var.worker_groups_launch_template_mixed[count.index], + "asg_max_size", + local.workers_group_defaults["asg_max_size"], + ) + min_size = lookup( + var.worker_groups_launch_template_mixed[count.index], + "asg_min_size", + local.workers_group_defaults["asg_min_size"], + ) + force_delete = lookup( + var.worker_groups_launch_template_mixed[count.index], + "asg_force_delete", + local.workers_group_defaults["asg_force_delete"], + ) + target_group_arns = lookup( + var.worker_groups_launch_template_mixed[count.index], + "target_group_arns", + local.workers_group_defaults["target_group_arns"] + ) + service_linked_role_arn = lookup( + var.worker_groups_launch_template_mixed[count.index], + "service_linked_role_arn", + local.workers_group_defaults["service_linked_role_arn"], + ) + vpc_zone_identifier = lookup( + var.worker_groups_launch_template_mixed[count.index], + "subnets", + local.workers_group_defaults["subnets"] + ) + protect_from_scale_in = lookup( + var.worker_groups_launch_template_mixed[count.index], + "protect_from_scale_in", + local.workers_group_defaults["protect_from_scale_in"], + ) + suspended_processes = lookup( + var.worker_groups_launch_template_mixed[count.index], + "suspended_processes", + local.workers_group_defaults["suspended_processes"] + ) + enabled_metrics = lookup( + var.worker_groups_launch_template_mixed[count.index], + "enabled_metrics", + local.workers_group_defaults["enabled_metrics"] + ) + placement_group = lookup( + var.worker_groups_launch_template_mixed[count.index], + "placement_group", + local.workers_group_defaults["placement_group"], + ) + termination_policies = lookup( + var.worker_groups_launch_template_mixed[count.index], + "termination_policies", + local.workers_group_defaults["termination_policies"] + ) mixed_instances_policy { instances_distribution { - on_demand_allocation_strategy = "${lookup(var.worker_groups_launch_template_mixed[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"])}" - on_demand_base_capacity = "${lookup(var.worker_groups_launch_template_mixed[count.index], "on_demand_base_capacity", local.workers_group_defaults["on_demand_base_capacity"])}" - on_demand_percentage_above_base_capacity = "${lookup(var.worker_groups_launch_template_mixed[count.index], "on_demand_percentage_above_base_capacity", local.workers_group_defaults["on_demand_percentage_above_base_capacity"])}" - spot_allocation_strategy = "${lookup(var.worker_groups_launch_template_mixed[count.index], "spot_allocation_strategy", local.workers_group_defaults["spot_allocation_strategy"])}" - spot_instance_pools = "${lookup(var.worker_groups_launch_template_mixed[count.index], "spot_instance_pools", local.workers_group_defaults["spot_instance_pools"])}" - spot_max_price = "${lookup(var.worker_groups_launch_template_mixed[count.index], "spot_max_price", local.workers_group_defaults["spot_max_price"])}" + on_demand_allocation_strategy = lookup( + var.worker_groups_launch_template_mixed[count.index], + "on_demand_allocation_strategy", + local.workers_group_defaults["on_demand_allocation_strategy"], + ) + on_demand_base_capacity = lookup( + var.worker_groups_launch_template_mixed[count.index], + "on_demand_base_capacity", + local.workers_group_defaults["on_demand_base_capacity"], + ) + on_demand_percentage_above_base_capacity = lookup( + var.worker_groups_launch_template_mixed[count.index], + "on_demand_percentage_above_base_capacity", + local.workers_group_defaults["on_demand_percentage_above_base_capacity"], + ) + spot_allocation_strategy = lookup( + var.worker_groups_launch_template_mixed[count.index], + "spot_allocation_strategy", + local.workers_group_defaults["spot_allocation_strategy"], + ) + spot_instance_pools = lookup( + var.worker_groups_launch_template_mixed[count.index], + "spot_instance_pools", + local.workers_group_defaults["spot_instance_pools"], + ) + spot_max_price = lookup( + var.worker_groups_launch_template_mixed[count.index], + "spot_max_price", + local.workers_group_defaults["spot_max_price"], + ) } launch_template { launch_template_specification { - launch_template_id = "${element(aws_launch_template.workers_launch_template_mixed.*.id, count.index)}" - version = "${lookup(var.worker_groups_launch_template_mixed[count.index], "launch_template_version", local.workers_group_defaults["launch_template_version"])}" + launch_template_id = aws_launch_template.workers_launch_template_mixed.*.id[count.index] + version = lookup( + var.worker_groups_launch_template_mixed[count.index], + "launch_template_version", + local.workers_group_defaults["launch_template_version"], + ) } - override { - instance_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "override_instance_type_1", local.workers_group_defaults["override_instance_type_1"])}" - } - - override { - instance_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "override_instance_type_2", local.workers_group_defaults["override_instance_type_2"])}" - } + dynamic "override" { + for_each = lookup( + var.worker_groups_launch_template_mixed[count.index], + "override_instance_types", + local.workers_group_defaults["override_instance_types"] + ) - override { - instance_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "override_instance_type_3", local.workers_group_defaults["override_instance_type_3"])}" + content { + instance_type = override.value + } } - override { - instance_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "override_instance_type_4", local.workers_group_defaults["override_instance_type_4"])}" - } } } - tags = ["${concat( - list( - map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true), - map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true), - map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups_launch_template_mixed[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled"}", "value", "true", "propagate_at_launch", false), - map("key", "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}", "value", "", "propagate_at_launch", false), - map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}Gi", "propagate_at_launch", false) - ), + tags = concat( + [ + { + "key" = "Name" + "value" = "${aws_eks_cluster.this.name}-${lookup( + var.worker_groups_launch_template_mixed[count.index], + "name", + count.index, + )}-eks_asg" + "propagate_at_launch" = true + }, + { + "key" = "kubernetes.io/cluster/${aws_eks_cluster.this.name}" + "value" = "owned" + "propagate_at_launch" = true + }, + { + "key" = "k8s.io/cluster-autoscaler/${lookup( + var.worker_groups_launch_template_mixed[count.index], + "autoscaling_enabled", + local.workers_group_defaults["autoscaling_enabled"], + ) ? "enabled" : "disabled"}" + "value" = "true" + "propagate_at_launch" = false + }, + { + "key" = "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}" + "value" = aws_eks_cluster.this.name + "propagate_at_launch" = false + }, + { + "key" = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" + "value" = "${lookup( + var.worker_groups_launch_template_mixed[count.index], + "root_volume_size", + local.workers_group_defaults["root_volume_size"], + )}Gi" + "propagate_at_launch" = false + }, + ], local.asg_tags, - var.worker_group_tags[contains(keys(var.worker_group_tags), "${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}") ? "${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}" : "default"]) - }"] + lookup( + var.worker_groups_launch_template_mixed[count.index], + "tags", + local.workers_group_defaults["tags"] + ) + ) lifecycle { create_before_destroy = true - ignore_changes = ["desired_capacity"] + ignore_changes = [desired_capacity] } } resource "aws_launch_template" "workers_launch_template_mixed" { - count = "${var.worker_group_launch_template_mixed_count}" - name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index)}" + count = local.worker_group_launch_template_mixed_count + name_prefix = "${aws_eks_cluster.this.name}-${lookup( + var.worker_groups_launch_template_mixed[count.index], + "name", + count.index, + )}" network_interfaces { - associate_public_ip_address = "${lookup(var.worker_groups_launch_template_mixed[count.index], "public_ip", local.workers_group_defaults["public_ip"])}" - delete_on_termination = "${lookup(var.worker_groups_launch_template_mixed[count.index], "eni_delete", local.workers_group_defaults["eni_delete"])}" - security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",", lookup(var.worker_groups_launch_template_mixed[count.index], "additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"] + associate_public_ip_address = lookup( + var.worker_groups_launch_template_mixed[count.index], + "public_ip", + local.workers_group_defaults["public_ip"], + ) + delete_on_termination = lookup( + var.worker_groups_launch_template_mixed[count.index], + "eni_delete", + local.workers_group_defaults["eni_delete"], + ) + security_groups = flatten([ + local.worker_security_group_id, + var.worker_additional_security_group_ids, + lookup( + var.worker_groups_launch_template_mixed[count.index], + "additional_security_group_ids", + local.workers_group_defaults["additional_security_group_ids"] + ) + ]) } iam_instance_profile { - name = "${element(coalescelist(aws_iam_instance_profile.workers_launch_template_mixed.*.name, data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.name), count.index)}" + name = coalescelist( + aws_iam_instance_profile.workers_launch_template_mixed.*.name, + data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.name, + )[count.index] } - image_id = "${lookup(var.worker_groups_launch_template_mixed[count.index], "ami_id", local.workers_group_defaults["ami_id"])}" - instance_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "instance_type", local.workers_group_defaults["instance_type"])}" - key_name = "${lookup(var.worker_groups_launch_template_mixed[count.index], "key_name", local.workers_group_defaults["key_name"])}" - user_data = "${base64encode(element(data.template_file.workers_launch_template_mixed.*.rendered, count.index))}" - ebs_optimized = "${lookup(var.worker_groups_launch_template_mixed[count.index], "ebs_optimized", lookup(local.ebs_optimized, lookup(var.worker_groups_launch_template_mixed[count.index], "instance_type", local.workers_group_defaults["instance_type"]), false))}" + image_id = lookup( + var.worker_groups_launch_template_mixed[count.index], + "ami_id", + local.workers_group_defaults["ami_id"], + ) + instance_type = lookup( + var.worker_groups_launch_template_mixed[count.index], + "instance_type", + local.workers_group_defaults["instance_type"], + ) + key_name = lookup( + var.worker_groups_launch_template_mixed[count.index], + "key_name", + local.workers_group_defaults["key_name"], + ) + user_data = base64encode( + data.template_file.workers_launch_template_mixed.*.rendered[count.index], + ) + ebs_optimized = lookup( + var.worker_groups_launch_template_mixed[count.index], + "ebs_optimized", + lookup( + local.ebs_optimized, + lookup( + var.worker_groups_launch_template_mixed[count.index], + "instance_type", + local.workers_group_defaults["instance_type"], + ), + false, + ), + ) monitoring { - enabled = "${lookup(var.worker_groups_launch_template_mixed[count.index], "enable_monitoring", local.workers_group_defaults["enable_monitoring"])}" + enabled = lookup( + var.worker_groups_launch_template_mixed[count.index], + "enable_monitoring", + local.workers_group_defaults["enable_monitoring"], + ) } placement { - tenancy = "${lookup(var.worker_groups_launch_template_mixed[count.index], "launch_template_placement_tenancy", local.workers_group_defaults["launch_template_placement_tenancy"])}" - group_name = "${lookup(var.worker_groups_launch_template_mixed[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])}" + tenancy = lookup( + var.worker_groups_launch_template_mixed[count.index], + "launch_template_placement_tenancy", + local.workers_group_defaults["launch_template_placement_tenancy"], + ) + group_name = lookup( + var.worker_groups_launch_template_mixed[count.index], + "launch_template_placement_group", + local.workers_group_defaults["launch_template_placement_group"], + ) } block_device_mappings { - device_name = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_block_device_name", local.workers_group_defaults["root_block_device_name"])}" + device_name = lookup( + var.worker_groups_launch_template_mixed[count.index], + "root_block_device_name", + local.workers_group_defaults["root_block_device_name"], + ) ebs { - volume_size = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}" - volume_type = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_volume_type", local.workers_group_defaults["root_volume_type"])}" - iops = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_iops", local.workers_group_defaults["root_iops"])}" - encrypted = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_encrypted", local.workers_group_defaults["root_encrypted"])}" - kms_key_id = "${lookup(var.worker_groups_launch_template_mixed[count.index], "root_kms_key_id", local.workers_group_defaults["root_kms_key_id"])}" + volume_size = lookup( + var.worker_groups_launch_template_mixed[count.index], + "root_volume_size", + local.workers_group_defaults["root_volume_size"], + ) + volume_type = lookup( + var.worker_groups_launch_template_mixed[count.index], + "root_volume_type", + local.workers_group_defaults["root_volume_type"], + ) + iops = lookup( + var.worker_groups_launch_template_mixed[count.index], + "root_iops", + local.workers_group_defaults["root_iops"], + ) + encrypted = lookup( + var.worker_groups_launch_template_mixed[count.index], + "root_encrypted", + local.workers_group_defaults["root_encrypted"], + ) + kms_key_id = lookup( + var.worker_groups_launch_template_mixed[count.index], + "root_kms_key_id", + local.workers_group_defaults["root_kms_key_id"], + ) delete_on_termination = true } } + tags = var.tags + lifecycle { create_before_destroy = true } } resource "aws_iam_instance_profile" "workers_launch_template_mixed" { - count = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_mixed_count : 0}" - name_prefix = "${aws_eks_cluster.this.name}" - role = "${lookup(var.worker_groups_launch_template_mixed[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}" - path = "${var.iam_path}" + count = var.manage_worker_iam_resources ? local.worker_group_launch_template_mixed_count : 0 + name_prefix = aws_eks_cluster.this.name + role = lookup( + var.worker_groups_launch_template_mixed[count.index], + "iam_role_id", + local.workers_group_defaults["iam_role_id"], + ) + path = var.iam_path } + From 9c3d222bd55fa8c37e48089ef59a5f35b91b8509 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Wed, 19 Jun 2019 17:27:25 +0200 Subject: [PATCH 021/962] New release: now supporting TF 0.12!! (#399) * New release: now supporting TF 0.12 * updating precommit hook version --- .pre-commit-config.yaml | 18 +++-------- CHANGELOG.md | 18 ++++++++--- README.md | 52 +++++++++++++++---------------- cluster.tf | 4 +-- examples/basic/main.tf | 6 ++-- examples/launch_templates/main.tf | 12 +++---- examples/spot_instances/main.tf | 26 ++++++++-------- version | 2 +- 8 files changed, 69 insertions(+), 69 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a7d2a9e49..ac4b62b453 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,5 @@ -repos: - - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.7.4 - hooks: - - id: terraform_fmt - - id: terraform_docs - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.0.0 - hooks: - - id: check-merge-conflict - - id: trailing-whitespace - - id: check-yaml - - id: check-added-large-files +- repo: git://github.com/antonbabenko/pre-commit-terraform + rev: v1.16.0 + hooks: + - id: terraform_fmt + - id: terraform_docs diff --git a/CHANGELOG.md b/CHANGELOG.md index a1829f9908..93a7cb6749 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,19 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v5.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...HEAD)] - 2019-06-??] +## [[v5.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...HEAD)] - 2019-06-??] + +### Added + + - Write your awesome addition here (by @you) + +### Changed + + - Write your awesome change here (by @you) + +# History + +## [[v5.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...v5.0.0)] - 2019-05-07] ### Added @@ -25,9 +37,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Change override instance_types to list (by @nauxliu on behalf of RightCapital) - Fix toggle for IAM instance profile creation for mixed launch templates (by @jnozo) -# History - -## [[v4.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1)] - 2019-05-07] +## [[v4.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.1...v4.0.2)] - 2019-05-07] ### Changed diff --git a/README.md b/README.md index 3d15e2695a..fd2a79ffd9 100644 --- a/README.md +++ b/README.md @@ -112,47 +112,47 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | string | `"true"` | no | +| cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | bool | `"true"` | no | | cluster\_create\_timeout | Timeout value when creating the EKS cluster. | string | `"15m"` | no | | cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | string | `"15m"` | no | -| cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | list | `[]` | no | -| cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | string | `"false"` | no | -| cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | string | `"true"` | no | +| cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | list(string) | `[]` | no | +| cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | bool | `"false"` | no | +| cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | bool | `"true"` | no | | cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. | string | `""` | no | -| cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days | string | `"90"` | no | +| cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers and provide API access to your current IP/32. | string | `""` | no | | cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.12"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Should end in a forward slash `/` . | string | `"./"` | no | | iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | -| kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list | `[]` | no | +| kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | | kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | string | `"aws-iam-authenticator"` | no | -| kubeconfig\_aws\_authenticator\_command\_args | Default arguments passed to the authenticator command. Defaults to [token -i $cluster_name]. | list | `[]` | no | -| kubeconfig\_aws\_authenticator\_env\_variables | Environment variables that should be used when executing the authenticator. e.g. { AWS_PROFILE = "eks"}. | map | `{}` | no | +| kubeconfig\_aws\_authenticator\_command\_args | Default arguments passed to the authenticator command. Defaults to [token -i $cluster_name]. | list(string) | `[]` | no | +| kubeconfig\_aws\_authenticator\_env\_variables | Environment variables that should be used when executing the authenticator. e.g. { AWS_PROFILE = "eks"}. | map(string) | `{}` | no | | kubeconfig\_name | Override the default name used for items kubeconfig. | string | `""` | no | -| local\_exec\_interpreter | Command to run for local-exec resources. Must be a shell-style interpreter. If you are on Windows Git Bash is a good choice. | list | `[ "/bin/sh", "-c" ]` | no | +| local\_exec\_interpreter | Command to run for local-exec resources. Must be a shell-style interpreter. If you are on Windows Git Bash is a good choice. | list(string) | `[ "/bin/sh", "-c" ]` | no | | manage\_aws\_auth | Whether to apply the aws-auth configmap file. | string | `"true"` | no | -| manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | string | `"true"` | no | -| manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | string | `"true"` | no | -| map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list | `[]` | no | -| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list | `[]` | no | -| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list | `[]` | no | +| manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | bool | `"true"` | no | +| manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | bool | `"true"` | no | +| map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | +| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(map(string)) | `[]` | no | +| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(map(string)) | `[]` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `""` | no | -| subnets | A list of subnets to place the EKS cluster and workers within. | list | n/a | yes | -| tags | A map of tags to add to all resources. | map | `{}` | no | +| subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | +| tags | A map of tags to add to all resources. | map(string) | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | -| worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list | `[]` | no | +| worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | | worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | -| worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | string | `"true"` | no | -| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | -| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | -| worker\_groups\_launch\_template\_mixed | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no | +| worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | +| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | +| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | +| worker\_groups\_launch\_template\_mixed | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | | worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster. | string | `""` | no | -| worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | string | `"1025"` | no | -| workers\_additional\_policies | Additional policies to be added to workers | list | `[]` | no | -| workers\_group\_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | map | `{}` | no | -| write\_aws\_auth\_config | Whether to write the aws-auth configmap file. | string | `"true"` | no | -| write\_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | string | `"true"` | no | +| worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | number | `"1025"` | no | +| workers\_additional\_policies | Additional policies to be added to workers | list(string) | `[]` | no | +| workers\_group\_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | any | `{}` | no | +| write\_aws\_auth\_config | Whether to write the aws-auth configmap file. | bool | `"true"` | no | +| write\_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | bool | `"true"` | no | ## Outputs diff --git a/cluster.tf b/cluster.tf index e39eaf5776..e49fafbfa9 100644 --- a/cluster.tf +++ b/cluster.tf @@ -1,8 +1,7 @@ resource "aws_cloudwatch_log_group" "this" { + count = length(var.cluster_enabled_log_types) > 0 ? 1 : 0 name = "/aws/eks/${var.cluster_name}/cluster" retention_in_days = var.cluster_log_retention_in_days - - count = length(var.cluster_enabled_log_types) > 0 ? 1 : 0 } resource "aws_eks_cluster" "this" { @@ -86,4 +85,3 @@ resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" role = aws_iam_role.cluster[0].name } - diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 9a25e4b835..b2d3ac55cd 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -71,8 +71,8 @@ resource "aws_security_group" "all_worker_mgmt" { } module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "2.6.0" + source = "terraform-aws-modules/vpc/aws" + version = "2.6.0" name = "test-vpc" cidr = "10.0.0.0/16" @@ -107,7 +107,7 @@ module "eks" { GithubOrg = "terraform-aws-modules" } - vpc_id = module.vpc.vpc_id + vpc_id = module.vpc.vpc_id worker_groups = [ { diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 27414a80cd..edc18e2503 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -24,8 +24,8 @@ resource "random_string" "suffix" { } module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "2.6.0" + source = "terraform-aws-modules/vpc/aws" + version = "2.6.0" name = "test-vpc-lt" cidr = "10.0.0.0/16" @@ -38,10 +38,10 @@ module "vpc" { } module "eks" { - source = "../.." - cluster_name = local.cluster_name - subnets = module.vpc.public_subnets - vpc_id = module.vpc.vpc_id + source = "../.." + cluster_name = local.cluster_name + subnets = module.vpc.public_subnets + vpc_id = module.vpc.vpc_id worker_groups_launch_template = [ { diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 9705be49c1..dc18c0e150 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -24,8 +24,8 @@ resource "random_string" "suffix" { } module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "2.6.0" + source = "terraform-aws-modules/vpc/aws" + version = "2.6.0" name = "test-vpc-spot" cidr = "10.0.0.0/16" @@ -38,20 +38,20 @@ module "vpc" { } module "eks" { - source = "../.." - cluster_name = local.cluster_name - subnets = module.vpc.public_subnets - vpc_id = module.vpc.vpc_id + source = "../.." + cluster_name = local.cluster_name + subnets = module.vpc.public_subnets + vpc_id = module.vpc.vpc_id worker_groups_launch_template_mixed = [ { - name = "spot-1" - override_instance_types = ["m5.large", "c5.large", "t3.large", "r5.large"] - spot_instance_pools = 4 - asg_max_size = 5 - asg_desired_capacity = 5 - kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" - public_ip = true + name = "spot-1" + override_instance_types = ["m5.large", "c5.large", "t3.large", "r5.large"] + spot_instance_pools = 4 + asg_max_size = 5 + asg_desired_capacity = 5 + kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" + public_ip = true }, ] } diff --git a/version b/version index bda368d509..d3845ad3c5 100644 --- a/version +++ b/version @@ -1 +1 @@ -v4.0.2 +v5.0.0 From 8260f1095e587713963b22a55fc829432dbadc8f Mon Sep 17 00:00:00 2001 From: Gauthier Date: Thu, 20 Jun 2019 12:18:12 +0200 Subject: [PATCH 022/962] EKS 1.13 out (#400) * Update to EKS 1.13 * Revert "Pin terraform version < 0.12" This reverts commit 4c71b12218378914c276715a85a61ac5aece9fa0. * Update providers in examples and format --- .travis.yml | 2 +- CHANGELOG.md | 1 + README.md | 2 +- examples/basic/main.tf | 13 ++++++++++++- examples/launch_templates/main.tf | 13 ++++++++++++- examples/spot_instances/main.tf | 13 ++++++++++++- variables.tf | 3 +-- 7 files changed, 40 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index c5fb533570..7954f76975 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ install: - bundle install before_script: -- export TERRAFORM_VERSION=0.12.2 +- export TERRAFORM_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version') - curl --silent --output terraform.zip "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - unzip terraform.zip ; rm -f terraform.zip; chmod +x terraform - mkdir -p ${HOME}/bin ; export PATH=${PATH}:${HOME}/bin; mv terraform ${HOME}/bin/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 93a7cb6749..e20a78b0de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Update EBS optimized instances type (by @gloutsch) - Added tagging for iam role created in `./cluster.tf` (@camilosantana) - Enable log retention for cloudwatch log groups (by @yuriipolishchuk) +- Update to EKS 1.13 (by @gloutsch) ### Changed diff --git a/README.md b/README.md index fd2a79ffd9..262fbb0c24 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers and provide API access to your current IP/32. | string | `""` | no | -| cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.12"` | no | +| cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.13"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Should end in a forward slash `/` . | string | `"./"` | no | | iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | | kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | diff --git a/examples/basic/main.tf b/examples/basic/main.tf index b2d3ac55cd..629c10bcde 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -11,6 +11,18 @@ provider "random" { version = "~> 2.1" } +provider "local" { + version = "~> 1.2" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + data "aws_availability_zones" "available" { } @@ -131,4 +143,3 @@ module "eks" { map_users = var.map_users map_accounts = var.map_accounts } - diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index edc18e2503..91cfd9837c 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -11,6 +11,18 @@ provider "random" { version = "~> 2.1" } +provider "local" { + version = "~> 1.2" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + data "aws_availability_zones" "available" { } @@ -58,4 +70,3 @@ module "eks" { }, ] } - diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index dc18c0e150..7c699c9325 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -11,6 +11,18 @@ provider "random" { version = "~> 2.1" } +provider "local" { + version = "~> 1.2" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + data "aws_availability_zones" "available" { } @@ -55,4 +67,3 @@ module "eks" { }, ] } - diff --git a/variables.tf b/variables.tf index 3d01ebb922..1ec8e48c87 100644 --- a/variables.tf +++ b/variables.tf @@ -24,7 +24,7 @@ variable "cluster_security_group_id" { variable "cluster_version" { description = "Kubernetes version to use for the EKS cluster." type = string - default = "1.12" + default = "1.13" } variable "config_output_path" { @@ -239,4 +239,3 @@ variable "manage_worker_iam_resources" { type = bool default = true } - From ba3377360e13e4773b27b1c5ec3dfd99b8ef949d Mon Sep 17 00:00:00 2001 From: till-krauss <40272510+till-krauss@users.noreply.github.com> Date: Mon, 24 Jun 2019 12:53:43 +0200 Subject: [PATCH 023/962] Added Option to use KMS key for cloudwatch logs group. (#404) --- CHANGELOG.md | 1 + README.md | 1 + cluster.tf | 1 + variables.tf | 6 +++++- 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e20a78b0de..ceb1a2cd8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added + - Option to set a KMS key for the log group and encrypt it (by @till-krauss) - Write your awesome addition here (by @you) ### Changed diff --git a/README.md b/README.md index 262fbb0c24..82a23218e4 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | bool | `"false"` | no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | bool | `"true"` | no | | cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. | string | `""` | no | +| cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | string | `""` | no | | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers and provide API access to your current IP/32. | string | `""` | no | diff --git a/cluster.tf b/cluster.tf index e49fafbfa9..f373eeebb1 100644 --- a/cluster.tf +++ b/cluster.tf @@ -2,6 +2,7 @@ resource "aws_cloudwatch_log_group" "this" { count = length(var.cluster_enabled_log_types) > 0 ? 1 : 0 name = "/aws/eks/${var.cluster_name}/cluster" retention_in_days = var.cluster_log_retention_in_days + kms_key_id = var.cluster_log_kms_key_id } resource "aws_eks_cluster" "this" { diff --git a/variables.tf b/variables.tf index 1ec8e48c87..34b41e69df 100644 --- a/variables.tf +++ b/variables.tf @@ -3,7 +3,11 @@ variable "cluster_enabled_log_types" { description = "A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html)" type = list(string) } - +variable "cluster_log_kms_key_id" { + default = "" + description = "If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html)" + type = string +} variable "cluster_log_retention_in_days" { default = 90 description = "Number of days to retain log events. Default retention - 90 days." From 069c2da770922f538979331a7e985bd65f3edeb5 Mon Sep 17 00:00:00 2001 From: Alex Romanov Date: Mon, 24 Jun 2019 06:41:40 -0700 Subject: [PATCH 024/962] typo (#407) --- local.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local.tf b/local.tf index d03360f0d4..97d93c438e 100644 --- a/local.tf +++ b/local.tf @@ -58,7 +58,7 @@ locals { protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. iam_role_id = local.default_iam_role_id # A custom IAM role id. Incompatible with iam_instance_profile_name. - suspended_processes = ["AZRebalance"] # A list of processes to to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] + suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] target_group_arns = [] # A list of ALB target group ARNs to be associated to the ASG enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] placement_group = "" # The name of the placement group into which to launch the instances, if any. From f1041af57c88820bc11605a2e8e630a4977f37d6 Mon Sep 17 00:00:00 2001 From: gbooth27 Date: Tue, 25 Jun 2019 08:04:01 -0700 Subject: [PATCH 025/962] add cloudwatch log group access (#410) * add cloudwatch log group access * update readme for outputs * update outputs * fix formatting * add splat for access to name --- CHANGELOG.md | 1 + README.md | 1 + outputs.tf | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ceb1a2cd8c..31f78e1510 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added - Option to set a KMS key for the log group and encrypt it (by @till-krauss) + - Output the name of the cloudwatch log group (by @gbooth27) - Write your awesome addition here (by @you) ### Changed diff --git a/README.md b/README.md index 82a23218e4..44d78cdb4d 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_id | The name/id of the EKS cluster. | | cluster\_security\_group\_id | Security group ID attached to the EKS cluster. | | cluster\_version | The Kubernetes server version for the EKS cluster. | +| cloudwatch\_log\_group\_name | The name of the cloudwatch log group created for the EKS cluster. | | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | diff --git a/outputs.tf b/outputs.tf index 1dfbfaae26..a65c76fe4a 100644 --- a/outputs.tf +++ b/outputs.tf @@ -43,6 +43,11 @@ output "cluster_iam_role_arn" { value = local.cluster_iam_role_arn } +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = aws_cloudwatch_log_group.this.*.name +} + output "kubeconfig" { description = "kubectl config file contents for this EKS cluster." value = data.template_file.kubeconfig.rendered From 80b0bc79af5d1218490d550d1c63f483e4948ee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=9B=B8=E8=BD=A9?= <9570112+nauxliu@users.noreply.github.com> Date: Thu, 27 Jun 2019 15:43:40 +0800 Subject: [PATCH 026/962] Remove outdated important issues (#416) --- docs/spot-instances.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/spot-instances.md b/docs/spot-instances.md index 23dba7886b..eaea7c1bac 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -88,6 +88,4 @@ Launch Template support is a recent addition to both AWS and this module. It mig ## Important issues -- https://github.com/terraform-aws-modules/terraform-aws-eks/issues/360 -- https://github.com/terraform-providers/terraform-provider-aws/issues/8475 - https://github.com/kubernetes/autoscaler/issues/1133 From 65901984e5d85a68c6a7f92afafc9bad115307ca Mon Sep 17 00:00:00 2001 From: Laurent Godet <1172558+lostick@users.noreply.github.com> Date: Fri, 5 Jul 2019 16:26:30 +0100 Subject: [PATCH 027/962] Fix suspended_processes to be a type list (#423) --- docs/spot-instances.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/spot-instances.md b/docs/spot-instances.md index eaea7c1bac..ef9df35bc6 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -34,7 +34,7 @@ Example worker group configuration that uses an ASG with launch configuration fo asg_max_size = 1 autoscaling_enabled = true kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=normal" - suspended_processes = "AZRebalance" + suspended_processes = ["AZRebalance"] }, { name = "spot-1" @@ -43,7 +43,7 @@ Example worker group configuration that uses an ASG with launch configuration fo asg_max_size = 20 autoscaling_enabled = true kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" - suspended_processes = "AZRebalance" + suspended_processes = ["AZRebalance"] }, { name = "spot-2" @@ -52,7 +52,7 @@ Example worker group configuration that uses an ASG with launch configuration fo asg_max_size = 20 autoscaling_enabled = true kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" - suspended_processes = "AZRebalance" + suspended_processes = ["AZRebalance"] } ] ``` @@ -69,7 +69,7 @@ Launch Template support is a recent addition to both AWS and this module. It mig asg_max_size = 10 autoscaling_enabled = true kubelet_extra_args = "--node-labels=spot=false" - suspended_processes = "AZRebalance" + suspended_processes = ["AZRebalance"] } ] From 4b1df0c0aff22dc44f4271e012a8fc32b5d02e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=9B=B8=E8=BD=A9?= <9570112+nauxliu@users.noreply.github.com> Date: Mon, 8 Jul 2019 17:18:13 +0800 Subject: [PATCH 028/962] Update default override instance types to work with Cluster Autoscaler (#425) --- CHANGELOG.md | 1 + docs/autoscaling.md | 4 ++++ examples/spot_instances/main.tf | 2 +- local.tf | 14 +++++++------- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31f78e1510..62d9ffc37c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed + - Update default override instance types to work with Cluster Autoscaler (by @nauxliu on behalf of RightCapital) - Write your awesome change here (by @you) # History diff --git a/docs/autoscaling.md b/docs/autoscaling.md index c572d08721..35ecc3154d 100644 --- a/docs/autoscaling.md +++ b/docs/autoscaling.md @@ -27,3 +27,7 @@ To install the chart, simply run helm with the `--values` option: ``` helm install stable/cluster-autoscaler --values=path/to/your/values-file.yaml ``` + +## See More + +[Using AutoScalingGroup MixedInstancesPolicy](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-autoscalinggroup-mixedinstancespolicy) \ No newline at end of file diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 7c699c9325..7db0d2298f 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -58,7 +58,7 @@ module "eks" { worker_groups_launch_template_mixed = [ { name = "spot-1" - override_instance_types = ["m5.large", "c5.large", "t3.large", "r5.large"] + override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] spot_instance_pools = 4 asg_max_size = 5 asg_desired_capacity = 5 diff --git a/local.tf b/local.tf index 97d93c438e..6caea574e7 100644 --- a/local.tf +++ b/local.tf @@ -73,13 +73,13 @@ locals { root_encrypted = "" # Whether the volume should be encrypted or not eni_delete = true # Delete the ENI on termination (if set to false you will have to manually delete before destroying) # Settings for launch templates with mixed instances policy - override_instance_types = ["m5.large", "c5.large", "t3.large", "r5.large"] # A list of override instance types for mixed instances policy - on_demand_allocation_strategy = "prioritized" # Strategy to use when launching on-demand instances. Valid values: prioritized. - on_demand_base_capacity = "0" # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances - on_demand_percentage_above_base_capacity = "0" # Percentage split between on-demand and Spot instances above the base on-demand capacity - spot_allocation_strategy = "lowest-price" # The only valid value is lowest-price, which is also the default value. The Auto Scaling group selects the cheapest Spot pools and evenly allocates your Spot capacity across the number of Spot pools that you specify. - spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." - spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price + override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] # A list of override instance types for mixed instances policy + on_demand_allocation_strategy = "prioritized" # Strategy to use when launching on-demand instances. Valid values: prioritized. + on_demand_base_capacity = "0" # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances + on_demand_percentage_above_base_capacity = "0" # Percentage split between on-demand and Spot instances above the base on-demand capacity + spot_allocation_strategy = "lowest-price" # The only valid value is lowest-price, which is also the default value. The Auto Scaling group selects the cheapest Spot pools and evenly allocates your Spot capacity across the number of Spot pools that you specify. + spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." + spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price } workers_group_defaults = merge( From 34b74518e8e996b4a54c1fc2975eac5f66feceee Mon Sep 17 00:00:00 2001 From: Alexander Shinkarenko Date: Fri, 12 Jul 2019 19:44:54 +0700 Subject: [PATCH 029/962] Configure 'cpu_credits' for workers (#253) (#426) * Configure 'cpu_credits' for workers (#253) * Updated CHANGELOG(#253) * Moved default to settings for launch templates(#253) --- CHANGELOG.md | 1 + local.tf | 1 + workers_launch_template.tf | 8 ++++++++ workers_launch_template_mixed.tf | 8 ++++++++ 4 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62d9ffc37c..03cd0a18ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Option to set a KMS key for the log group and encrypt it (by @till-krauss) - Output the name of the cloudwatch log group (by @gbooth27) + - Added `cpu_credits` param for the workers defined in `worker_groups_launch_template` (by @a-shink) - Write your awesome addition here (by @you) ### Changed diff --git a/local.tf b/local.tf index 6caea574e7..8cf58fda04 100644 --- a/local.tf +++ b/local.tf @@ -72,6 +72,7 @@ locals { launch_template_placement_group = "" # The name of the placement group into which to launch the instances, if any. root_encrypted = "" # Whether the volume should be encrypted or not eni_delete = true # Delete the ENI on termination (if set to false you will have to manually delete before destroying) + cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs # Settings for launch templates with mixed instances policy override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] # A list of override instance types for mixed instances policy on_demand_allocation_strategy = "prioritized" # Strategy to use when launching on-demand instances. Valid values: prioritized. diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 931c18cea3..b57f4e6ed3 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -200,6 +200,14 @@ resource "aws_launch_template" "workers_launch_template" { ), ) + credit_specification { + cpu_credits = lookup( + var.worker_groups_launch_template[count.index], + "cpu_credits", + local.workers_group_defaults["cpu_credits"] + ) + } + monitoring { enabled = lookup( var.worker_groups_launch_template[count.index], diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf index 7de2c9a09e..7dd9d909ba 100644 --- a/workers_launch_template_mixed.tf +++ b/workers_launch_template_mixed.tf @@ -250,6 +250,14 @@ resource "aws_launch_template" "workers_launch_template_mixed" { ), ) + credit_specification { + cpu_credits = lookup( + var.worker_groups_launch_template_mixed[count.index], + "cpu_credits", + local.workers_group_defaults["cpu_credits"] + ) + } + monitoring { enabled = lookup( var.worker_groups_launch_template_mixed[count.index], From ec64a7401976c836554506eb7b9b58c1b3d150d1 Mon Sep 17 00:00:00 2001 From: mandarin801 Date: Wed, 17 Jul 2019 13:56:54 +0300 Subject: [PATCH 030/962] Update README.md (#436) fix Error --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 44d78cdb4d..fe9fb1f569 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,11 @@ module "my-cluster" { { instance_type = "m4.large" asg_max_size = 5 - tags = { + tags = [{ key = "foo" value = "bar" propagate_at_launch = true - } + }] } ] From a95ad00b0a9a0ca5e519dab5b07bcd6d827aec6a Mon Sep 17 00:00:00 2001 From: Onur Sam <51121022+onur-sam-gtn-ai@users.noreply.github.com> Date: Tue, 23 Jul 2019 16:33:16 +0100 Subject: [PATCH 031/962] added market options to request spot instances without pools (#409) * added market options to request spot instances without specifying pools in a launch template * formatted * simplified implementation * formatted * Update CHANGELOG.md --- CHANGELOG.md | 1 + local.tf | 1 + workers_launch_template.tf | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03cd0a18ce..ad9eb3b171 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Option to set a KMS key for the log group and encrypt it (by @till-krauss) - Output the name of the cloudwatch log group (by @gbooth27) + - Option to use spot instances with launch templates without defining pools, especially useful for GPU instance types (@onur-sam-gtn-ai) - Added `cpu_credits` param for the workers defined in `worker_groups_launch_template` (by @a-shink) - Write your awesome addition here (by @you) diff --git a/local.tf b/local.tf index 8cf58fda04..e2f8b3a75f 100644 --- a/local.tf +++ b/local.tf @@ -81,6 +81,7 @@ locals { spot_allocation_strategy = "lowest-price" # The only valid value is lowest-price, which is also the default value. The Auto Scaling group selects the cheapest Spot pools and evenly allocates your Spot capacity across the number of Spot pools that you specify. spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price + market_type = null # set to "spot" to use spot instances without defining pools } workers_group_defaults = merge( diff --git a/workers_launch_template.tf b/workers_launch_template.tf index b57f4e6ed3..caee563387 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -229,6 +229,14 @@ resource "aws_launch_template" "workers_launch_template" { ) } + instance_market_options { + market_type = lookup( + var.worker_groups_launch_template[count.index], + "market_type", + local.workers_group_defaults["market_type"], + ) + } + block_device_mappings { device_name = lookup( var.worker_groups_launch_template[count.index], From f755300550e0f1975d18f415e4827003a257dc45 Mon Sep 17 00:00:00 2001 From: James <40496186+kuwas@users.noreply.github.com> Date: Tue, 23 Jul 2019 11:35:36 -0400 Subject: [PATCH 032/962] chore: fix README.md worker_groups tags syntax (#405) From e876ce20b9b64b434b33ffcad9cc0141dbc68feb Mon Sep 17 00:00:00 2001 From: Karoline Pauls <43616133+karolinepauls@users.noreply.github.com> Date: Tue, 30 Jul 2019 14:05:08 +0100 Subject: [PATCH 033/962] VPC: `enable_dns_hostnames = true` in examples (#446) --- CHANGELOG.md | 1 + README.md | 4 ++-- examples/basic/main.tf | 15 ++++++++------- examples/launch_templates/main.tf | 9 +++++---- examples/spot_instances/main.tf | 9 +++++---- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad9eb3b171..02eee5ddd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed - Update default override instance types to work with Cluster Autoscaler (by @nauxliu on behalf of RightCapital) + - Examples now specify `enable_dns_hostnames = true`, as per [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html) (by @karolinepauls) - Write your awesome change here (by @you) # History diff --git a/README.md b/README.md index fe9fb1f569..011e697509 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aw * You want to create an EKS cluster and an autoscaling group of workers for the cluster. * You want these resources to exist within security groups that allow communication and coordination. These can be user provided or created within the module. -* You've created a Virtual Private Cloud (VPC) and subnets where you intend to put the EKS resources. +* You've created a Virtual Private Cloud (VPC) and subnets where you intend to put the EKS resources. The VPC satisfies [EKS requirements](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html). * If `manage_aws_auth = true`, it's required that both [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) (>=1.10) and [`aws-iam-authenticator`](https://github.com/kubernetes-sigs/aws-iam-authenticator#4-set-up-kubectl-to-use-authentication-tokens-provided-by-aws-iam-authenticator-for-kubernetes) are installed and on your shell's PATH. ## Usage example @@ -141,7 +141,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `""` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | | tags | A map of tags to add to all resources. | map(string) | `{}` | no | -| vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | +| vpc\_id | VPC where the cluster and workers will be deployed. It must satisfy [EKS requirements](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html). | string | n/a | yes | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | | worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 629c10bcde..c7d0cadc00 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -86,13 +86,14 @@ module "vpc" { source = "terraform-aws-modules/vpc/aws" version = "2.6.0" - name = "test-vpc" - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true + name = "test-vpc" + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true tags = { "kubernetes.io/cluster/${local.cluster_name}" = "shared" diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 91cfd9837c..51148aac01 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -39,10 +39,11 @@ module "vpc" { source = "terraform-aws-modules/vpc/aws" version = "2.6.0" - name = "test-vpc-lt" - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + name = "test-vpc-lt" + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_dns_hostnames = true tags = { "kubernetes.io/cluster/${local.cluster_name}" = "shared" diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 7db0d2298f..c8550e21d2 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -39,10 +39,11 @@ module "vpc" { source = "terraform-aws-modules/vpc/aws" version = "2.6.0" - name = "test-vpc-spot" - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + name = "test-vpc-spot" + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_dns_hostnames = true tags = { "kubernetes.io/cluster/${local.cluster_name}" = "shared" From 52286a00653cf695835ec4c22dc090417358bf21 Mon Sep 17 00:00:00 2001 From: Sergiu Plotnicu <33776238+sppwf@users.noreply.github.com> Date: Tue, 30 Jul 2019 16:17:49 +0300 Subject: [PATCH 034/962] Support for tagging EBS Volumes created by "workers_launch_template*.tf" (#450) * Added support to tag EBS volumes used by EKS nodes created by workers_launch_template*.tf * Did terraform fmt on workers_launch_template*.tf --- CHANGELOG.md | 1 + workers_launch_template.tf | 15 +++++++++++++++ workers_launch_template_mixed.tf | 15 +++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02eee5ddd9..e38c115aac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Output the name of the cloudwatch log group (by @gbooth27) - Option to use spot instances with launch templates without defining pools, especially useful for GPU instance types (@onur-sam-gtn-ai) - Added `cpu_credits` param for the workers defined in `worker_groups_launch_template` (by @a-shink) + - Added support for EBS Volumes tag in `worker_groups_launch_template` and `workers_launch_template_mixed.tf` (by @sppwf) - Write your awesome addition here (by @you) ### Changed diff --git a/workers_launch_template.tf b/workers_launch_template.tf index caee563387..1c4fe77722 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -274,6 +274,21 @@ resource "aws_launch_template" "workers_launch_template" { } } + tag_specifications { + resource_type = "volume" + + tags = merge( + { + "Name" = "${aws_eks_cluster.this.name}-${lookup( + var.worker_groups_launch_template[count.index], + "name", + count.index, + )}-eks_asg" + }, + var.tags, + ) + } + tags = var.tags lifecycle { diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf index 7dd9d909ba..6e1d62013d 100644 --- a/workers_launch_template_mixed.tf +++ b/workers_launch_template_mixed.tf @@ -316,6 +316,21 @@ resource "aws_launch_template" "workers_launch_template_mixed" { } } + tag_specifications { + resource_type = "volume" + + tags = merge( + { + "Name" = "${aws_eks_cluster.this.name}-${lookup( + var.worker_groups_launch_template[count.index], + "name", + count.index, + )}-eks_asg" + }, + var.tags, + ) + } + tags = var.tags lifecycle { From 6ea3582940c0312b5031c8c9c8dafb1e86a98ceb Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 30 Jul 2019 16:02:21 +0200 Subject: [PATCH 035/962] new release (#452) --- CHANGELOG.md | 21 ++++++++++++++------- README.md | 4 ++-- local.tf | 2 -- version | 2 +- workers_launch_template.tf | 9 --------- workers_launch_template_mixed.tf | 3 +-- 6 files changed, 18 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e38c115aac..c4e1aa9536 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,26 +7,33 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v5.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...HEAD)] - 2019-06-??] +## [[v5.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...HEAD)] - 2019-08-??] + +### Added + + - Write your awesome addition here (by @you) + +### Changed + + - Write your awesome change here (by @you) + +# History + +## [[v5.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...v5.1.0)] - 2019-07-30] ### Added - Option to set a KMS key for the log group and encrypt it (by @till-krauss) - Output the name of the cloudwatch log group (by @gbooth27) - - Option to use spot instances with launch templates without defining pools, especially useful for GPU instance types (@onur-sam-gtn-ai) - Added `cpu_credits` param for the workers defined in `worker_groups_launch_template` (by @a-shink) - Added support for EBS Volumes tag in `worker_groups_launch_template` and `workers_launch_template_mixed.tf` (by @sppwf) - - Write your awesome addition here (by @you) ### Changed - Update default override instance types to work with Cluster Autoscaler (by @nauxliu on behalf of RightCapital) - Examples now specify `enable_dns_hostnames = true`, as per [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html) (by @karolinepauls) - - Write your awesome change here (by @you) - -# History -## [[v5.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...v5.0.0)] - 2019-05-07] +## [[v5.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...v5.0.0)] - 2019-06-19] ### Added diff --git a/README.md b/README.md index 011e697509..3c3e782774 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `""` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | | tags | A map of tags to add to all resources. | map(string) | `{}` | no | -| vpc\_id | VPC where the cluster and workers will be deployed. It must satisfy [EKS requirements](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html). | string | n/a | yes | +| vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | | worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | @@ -159,6 +159,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | |------|-------------| +| cloudwatch\_log\_group\_name | Name of cloudwatch log group created | | cluster\_arn | The Amazon Resource Name (ARN) of the cluster. | | cluster\_certificate\_authority\_data | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | | cluster\_endpoint | The endpoint for your EKS Kubernetes API. | @@ -167,7 +168,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_id | The name/id of the EKS cluster. | | cluster\_security\_group\_id | Security group ID attached to the EKS cluster. | | cluster\_version | The Kubernetes server version for the EKS cluster. | -| cloudwatch\_log\_group\_name | The name of the cloudwatch log group created for the EKS cluster. | | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | diff --git a/local.tf b/local.tf index e2f8b3a75f..915c8b274f 100644 --- a/local.tf +++ b/local.tf @@ -81,7 +81,6 @@ locals { spot_allocation_strategy = "lowest-price" # The only valid value is lowest-price, which is also the default value. The Auto Scaling group selects the cheapest Spot pools and evenly allocates your Spot capacity across the number of Spot pools that you specify. spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price - market_type = null # set to "spot" to use spot instances without defining pools } workers_group_defaults = merge( @@ -241,4 +240,3 @@ locals { "x1e.32xlarge" = true } } - diff --git a/version b/version index d3845ad3c5..6b7ace3e91 100644 --- a/version +++ b/version @@ -1 +1 @@ -v5.0.0 +v5.1.0 diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 1c4fe77722..44bcd212c2 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -229,14 +229,6 @@ resource "aws_launch_template" "workers_launch_template" { ) } - instance_market_options { - market_type = lookup( - var.worker_groups_launch_template[count.index], - "market_type", - local.workers_group_defaults["market_type"], - ) - } - block_device_mappings { device_name = lookup( var.worker_groups_launch_template[count.index], @@ -306,4 +298,3 @@ resource "aws_iam_instance_profile" "workers_launch_template" { ) path = var.iam_path } - diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf index 6e1d62013d..61df13670d 100644 --- a/workers_launch_template_mixed.tf +++ b/workers_launch_template_mixed.tf @@ -322,7 +322,7 @@ resource "aws_launch_template" "workers_launch_template_mixed" { tags = merge( { "Name" = "${aws_eks_cluster.this.name}-${lookup( - var.worker_groups_launch_template[count.index], + var.worker_groups_launch_template_mixed[count.index], "name", count.index, )}-eks_asg" @@ -348,4 +348,3 @@ resource "aws_iam_instance_profile" "workers_launch_template_mixed" { ) path = var.iam_path } - From 6d0025ea03e3d5216aa15da8e8006bafd0c1925e Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 30 Jul 2019 16:08:56 +0200 Subject: [PATCH 036/962] move/merge docs (#453) --- examples/README.md | 114 ++++++++++++++++++++++++++ examples/eks_test_fixture/README.md | 121 ---------------------------- 2 files changed, 114 insertions(+), 121 deletions(-) delete mode 100644 examples/eks_test_fixture/README.md diff --git a/examples/README.md b/examples/README.md index d1da18ab43..1217618d98 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,3 +5,117 @@ These serve a few purposes: 1. Shows developers how to use the module in a straightforward way as integrated with other terraform community supported modules. 2. Serves as the test infrastructure for CI on the project. 3. Provides a simple way to play with the Kubernetes cluster you create. + +## IAM Permissions + +The following IAM policy is the minimum needed to execute the module from the test suite. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "autoscaling:AttachInstances", + "autoscaling:CreateAutoScalingGroup", + "autoscaling:CreateLaunchConfiguration", + "autoscaling:CreateOrUpdateTags", + "autoscaling:DeleteAutoScalingGroup", + "autoscaling:DeleteLaunchConfiguration", + "autoscaling:DeleteTags", + "autoscaling:Describe*", + "autoscaling:DetachInstances", + "autoscaling:SetDesiredCapacity", + "autoscaling:UpdateAutoScalingGroup", + "autoscaling:SuspendProcesses", + "ec2:AllocateAddress", + "ec2:AssignPrivateIpAddresses", + "ec2:Associate*", + "ec2:AttachInternetGateway", + "ec2:AttachNetworkInterface", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateDefaultSubnet", + "ec2:CreateDhcpOptions", + "ec2:CreateEgressOnlyInternetGateway", + "ec2:CreateInternetGateway", + "ec2:CreateNatGateway", + "ec2:CreateNetworkInterface", + "ec2:CreateRoute", + "ec2:CreateRouteTable", + "ec2:CreateSecurityGroup", + "ec2:CreateSubnet", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:CreateVpc", + "ec2:DeleteDhcpOptions", + "ec2:DeleteEgressOnlyInternetGateway", + "ec2:DeleteInternetGateway", + "ec2:DeleteNatGateway", + "ec2:DeleteNetworkInterface", + "ec2:DeleteRoute", + "ec2:DeleteRouteTable", + "ec2:DeleteSecurityGroup", + "ec2:DeleteSubnet", + "ec2:DeleteTags", + "ec2:DeleteVolume", + "ec2:DeleteVpc", + "ec2:DeleteVpnGateway", + "ec2:Describe*", + "ec2:DetachInternetGateway", + "ec2:DetachNetworkInterface", + "ec2:DetachVolume", + "ec2:Disassociate*", + "ec2:ModifySubnetAttribute", + "ec2:ModifyVpcAttribute", + "ec2:ModifyVpcEndpoint", + "ec2:ReleaseAddress", + "ec2:RevokeSecurityGroupEgress", + "ec2:RevokeSecurityGroupIngress", + "ec2:UpdateSecurityGroupRuleDescriptionsEgress", + "ec2:UpdateSecurityGroupRuleDescriptionsIngress", + "ec2:CreateLaunchTemplate", + "ec2:CreateLaunchTemplateVersion", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:GetLaunchTemplateData", + "ec2:ModifyLaunchTemplate", + "ec2:RunInstances", + "eks:CreateCluster", + "eks:DeleteCluster", + "eks:DescribeCluster", + "eks:ListClusters", + "eks:UpdateClusterConfig", + "eks:DescribeUpdate", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreatePolicyVersion", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeleteRole", + "iam:DeleteRolePolicy", + "iam:DeleteServiceLinkedRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:List*", + "iam:PassRole", + "iam:PutRolePolicy", + "iam:RemoveRoleFromInstanceProfile", + "iam:UpdateAssumeRolePolicy" + ], + "Resource": "*" + } + ] +} +``` diff --git a/examples/eks_test_fixture/README.md b/examples/eks_test_fixture/README.md deleted file mode 100644 index 20c166a270..0000000000 --- a/examples/eks_test_fixture/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# eks_test_fixture example - -This set of templates serves a few purposes. It: - -1. shows developers how to use the module in a straightforward way as integrated with other terraform community supported modules. -2. serves as the test infrastructure for CI on the project. -3. provides a simple way to play with the Kubernetes cluster you create. - -## IAM Permissions - -The following IAM policy is the minimum needed to execute the module from the test suite. - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "autoscaling:AttachInstances", - "autoscaling:CreateAutoScalingGroup", - "autoscaling:CreateLaunchConfiguration", - "autoscaling:CreateOrUpdateTags", - "autoscaling:DeleteAutoScalingGroup", - "autoscaling:DeleteLaunchConfiguration", - "autoscaling:DeleteTags", - "autoscaling:Describe*", - "autoscaling:DetachInstances", - "autoscaling:SetDesiredCapacity", - "autoscaling:UpdateAutoScalingGroup", - "autoscaling:SuspendProcesses", - "ec2:AllocateAddress", - "ec2:AssignPrivateIpAddresses", - "ec2:Associate*", - "ec2:AttachInternetGateway", - "ec2:AttachNetworkInterface", - "ec2:AuthorizeSecurityGroupEgress", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateDefaultSubnet", - "ec2:CreateDhcpOptions", - "ec2:CreateEgressOnlyInternetGateway", - "ec2:CreateInternetGateway", - "ec2:CreateNatGateway", - "ec2:CreateNetworkInterface", - "ec2:CreateRoute", - "ec2:CreateRouteTable", - "ec2:CreateSecurityGroup", - "ec2:CreateSubnet", - "ec2:CreateTags", - "ec2:CreateVolume", - "ec2:CreateVpc", - "ec2:DeleteDhcpOptions", - "ec2:DeleteEgressOnlyInternetGateway", - "ec2:DeleteInternetGateway", - "ec2:DeleteNatGateway", - "ec2:DeleteNetworkInterface", - "ec2:DeleteRoute", - "ec2:DeleteRouteTable", - "ec2:DeleteSecurityGroup", - "ec2:DeleteSubnet", - "ec2:DeleteTags", - "ec2:DeleteVolume", - "ec2:DeleteVpc", - "ec2:DeleteVpnGateway", - "ec2:Describe*", - "ec2:DetachInternetGateway", - "ec2:DetachNetworkInterface", - "ec2:DetachVolume", - "ec2:Disassociate*", - "ec2:ModifySubnetAttribute", - "ec2:ModifyVpcAttribute", - "ec2:ModifyVpcEndpoint", - "ec2:ReleaseAddress", - "ec2:RevokeSecurityGroupEgress", - "ec2:RevokeSecurityGroupIngress", - "ec2:UpdateSecurityGroupRuleDescriptionsEgress", - "ec2:UpdateSecurityGroupRuleDescriptionsIngress", - "ec2:CreateLaunchTemplate", - "ec2:CreateLaunchTemplateVersion", - "ec2:DeleteLaunchTemplate", - "ec2:DeleteLaunchTemplateVersions", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeLaunchTemplateVersions", - "ec2:GetLaunchTemplateData", - "ec2:ModifyLaunchTemplate", - "ec2:RunInstances", - "eks:CreateCluster", - "eks:DeleteCluster", - "eks:DescribeCluster", - "eks:ListClusters", - "eks:UpdateClusterConfig", - "eks:DescribeUpdate", - "iam:AddRoleToInstanceProfile", - "iam:AttachRolePolicy", - "iam:CreateInstanceProfile", - "iam:CreatePolicy", - "iam:CreatePolicyVersion", - "iam:CreateRole", - "iam:DeleteInstanceProfile", - "iam:DeletePolicy", - "iam:DeleteRole", - "iam:DeleteRolePolicy", - "iam:DeleteServiceLinkedRole", - "iam:DetachRolePolicy", - "iam:GetInstanceProfile", - "iam:GetPolicy", - "iam:GetPolicyVersion", - "iam:GetRole", - "iam:GetRolePolicy", - "iam:List*", - "iam:PassRole", - "iam:PutRolePolicy", - "iam:RemoveRoleFromInstanceProfile", - "iam:UpdateAssumeRolePolicy" - ], - "Resource": "*" - } - ] -} -``` From c5c3d385b7f66fac1e7481e992879aa0785c994d Mon Sep 17 00:00:00 2001 From: Alexandr Grab Date: Thu, 1 Aug 2019 17:26:41 +0300 Subject: [PATCH 037/962] Additional tag for autoscaling enabled (#454) * Additional tag for autoscaling_enabled * Added changelog --- CHANGELOG.md | 6 ++++++ workers.tf | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4e1aa9536..a2fb0cf389 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,12 @@ project adheres to [Semantic Versioning](http://semver.org/). # History +## [[v5.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v5.1.1)] - 2019-07-30] + +### Added + + - Added new tag in `worker.tf` with autoscaling_enabled = true flag (by @insider89) + ## [[v5.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...v5.1.0)] - 2019-07-30] ### Added diff --git a/workers.tf b/workers.tf index 93a945c507..c50e1bc5ef 100644 --- a/workers.tf +++ b/workers.tf @@ -77,6 +77,11 @@ resource "aws_autoscaling_group" "workers" { "value" = "owned" "propagate_at_launch" = true }, + { + "key" = "k8s.io/cluster/${aws_eks_cluster.this.name}" + "value" = "owned" + "propagate_at_launch" = true + }, { "key" = "k8s.io/cluster-autoscaler/${lookup( var.worker_groups[count.index], From 630a0cc3fbd14ee1ba506ae22bd100929772148a Mon Sep 17 00:00:00 2001 From: Karoline Pauls <43616133+karolinepauls@users.noreply.github.com> Date: Tue, 6 Aug 2019 16:26:05 +0100 Subject: [PATCH 038/962] Fix fmt for v0.12.6 (#460) --- aws_auth.tf | 18 +++++++++--------- data.tf | 22 +++++++++++----------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/aws_auth.tf b/aws_auth.tf index c5cb532509..6d3179a6b8 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -27,8 +27,8 @@ EOS triggers = { kube_config_map_rendered = data.template_file.kubeconfig.rendered - config_map_rendered = data.template_file.config_map_aws_auth.rendered - endpoint = aws_eks_cluster.this.endpoint + config_map_rendered = data.template_file.config_map_aws_auth.rendered + endpoint = aws_eks_cluster.this.endpoint } } @@ -36,7 +36,7 @@ data "aws_caller_identity" "current" { } data "template_file" "launch_template_mixed_worker_role_arns" { - count = local.worker_group_launch_template_mixed_count + count = local.worker_group_launch_template_mixed_count template = file("${path.module}/templates/worker-role.tpl") vars = { @@ -51,7 +51,7 @@ data "template_file" "launch_template_mixed_worker_role_arns" { } data "template_file" "launch_template_worker_role_arns" { - count = local.worker_group_launch_template_count + count = local.worker_group_launch_template_count template = file("${path.module}/templates/worker-role.tpl") vars = { @@ -66,7 +66,7 @@ data "template_file" "launch_template_worker_role_arns" { } data "template_file" "worker_role_arns" { - count = local.worker_group_count + count = local.worker_group_count template = file("${path.module}/templates/worker-role.tpl") vars = { @@ -95,8 +95,8 @@ data "template_file" "config_map_aws_auth" { ), ), ) - map_users = join("", data.template_file.map_users.*.rendered) - map_roles = join("", data.template_file.map_roles.*.rendered) + map_users = join("", data.template_file.map_users.*.rendered) + map_roles = join("", data.template_file.map_roles.*.rendered) map_accounts = join("", data.template_file.map_accounts.*.rendered) } } @@ -110,7 +110,7 @@ data "template_file" "map_users" { vars = { user_arn = var.map_users[count.index]["user_arn"] username = var.map_users[count.index]["username"] - group = var.map_users[count.index]["group"] + group = var.map_users[count.index]["group"] } } @@ -123,7 +123,7 @@ data "template_file" "map_roles" { vars = { role_arn = var.map_roles[count.index]["role_arn"] username = var.map_roles[count.index]["username"] - group = var.map_roles[count.index]["group"] + group = var.map_roles[count.index]["group"] } } diff --git a/data.tf b/data.tf index f7f7a33233..01b1145fad 100644 --- a/data.tf +++ b/data.tf @@ -81,17 +81,17 @@ EOF vars = { value = values(var.kubeconfig_aws_authenticator_env_variables)[count.index] - key = keys(var.kubeconfig_aws_authenticator_env_variables)[count.index] + key = keys(var.kubeconfig_aws_authenticator_env_variables)[count.index] } } data "template_file" "userdata" { - count = local.worker_group_count + count = local.worker_group_count template = file("${path.module}/templates/userdata.sh.tpl") vars = { - cluster_name = aws_eks_cluster.this.name - endpoint = aws_eks_cluster.this.endpoint + cluster_name = aws_eks_cluster.this.name + endpoint = aws_eks_cluster.this.endpoint cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data pre_userdata = lookup( var.worker_groups[count.index], @@ -117,12 +117,12 @@ data "template_file" "userdata" { } data "template_file" "launch_template_userdata" { - count = local.worker_group_launch_template_count + count = local.worker_group_launch_template_count template = file("${path.module}/templates/userdata.sh.tpl") vars = { - cluster_name = aws_eks_cluster.this.name - endpoint = aws_eks_cluster.this.endpoint + cluster_name = aws_eks_cluster.this.name + endpoint = aws_eks_cluster.this.endpoint cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data pre_userdata = lookup( var.worker_groups_launch_template[count.index], @@ -148,12 +148,12 @@ data "template_file" "launch_template_userdata" { } data "template_file" "workers_launch_template_mixed" { - count = local.worker_group_launch_template_mixed_count + count = local.worker_group_launch_template_mixed_count template = file("${path.module}/templates/userdata.sh.tpl") vars = { - cluster_name = aws_eks_cluster.this.name - endpoint = aws_eks_cluster.this.endpoint + cluster_name = aws_eks_cluster.this.name + endpoint = aws_eks_cluster.this.endpoint cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data pre_userdata = lookup( var.worker_groups_launch_template_mixed[count.index], @@ -180,7 +180,7 @@ data "template_file" "workers_launch_template_mixed" { data "aws_iam_role" "custom_cluster_iam_role" { count = var.manage_cluster_iam_resources ? 0 : 1 - name = var.cluster_iam_role_name + name = var.cluster_iam_role_name } data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" { From ac62edc8722c0615dc40df8cd9493ff72f502497 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Tue, 6 Aug 2019 17:46:48 +0200 Subject: [PATCH 039/962] Add `required_providers` to set minimum versions (#464) --- CHANGELOG.md | 1 + versions.tf | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2fb0cf389..ce9bf6bb3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added - Write your awesome addition here (by @you) + - Added `required_providers` to enforce provider minimum versions (by @dpiddockcmp) ### Changed diff --git a/versions.tf b/versions.tf index ac97c6ac8e..affd035db0 100644 --- a/versions.tf +++ b/versions.tf @@ -1,4 +1,10 @@ - terraform { required_version = ">= 0.12" + + required_providers { + aws = ">= 2.8" + local = ">= 1.2" + null = ">= 2.1" + template = ">= 2.1" + } } From ebac6c92bfa362f67df228ee1d6462426d34f082 Mon Sep 17 00:00:00 2001 From: Lucas Giacomazzi Date: Tue, 6 Aug 2019 13:01:01 -0300 Subject: [PATCH 040/962] Adding tags for Log groups and workers IAM role (#448) * Adding tags for Cloudwatch log group * Adding tags for workers IAM role * Update CHANGELOG.md * Removing change in CHANGELOG.md * Fixing formatting issues --- CHANGELOG.md | 2 ++ cluster.tf | 1 + workers.tf | 1 + 3 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce9bf6bb3b..ee4970c597 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added + - Added support for log group tag in `./cluster.tf` (@lucas-giaco) + - Added support for workers iam role tag in `./workers.tf` (@lucas-giaco) - Write your awesome addition here (by @you) - Added `required_providers` to enforce provider minimum versions (by @dpiddockcmp) diff --git a/cluster.tf b/cluster.tf index f373eeebb1..7fa9409f33 100644 --- a/cluster.tf +++ b/cluster.tf @@ -3,6 +3,7 @@ resource "aws_cloudwatch_log_group" "this" { name = "/aws/eks/${var.cluster_name}/cluster" retention_in_days = var.cluster_log_retention_in_days kms_key_id = var.cluster_log_kms_key_id + tags = var.tags } resource "aws_eks_cluster" "this" { diff --git a/workers.tf b/workers.tf index c50e1bc5ef..8a330d4a3f 100644 --- a/workers.tf +++ b/workers.tf @@ -286,6 +286,7 @@ resource "aws_iam_role" "workers" { permissions_boundary = var.permissions_boundary path = var.iam_path force_detach_policies = true + tags = var.tags } resource "aws_iam_instance_profile" "workers" { From c9986f5e01c785875cb1e9cfa21ba195ef1bbab7 Mon Sep 17 00:00:00 2001 From: Petri Kero Date: Tue, 6 Aug 2019 19:05:54 +0300 Subject: [PATCH 041/962] Fix errors from usage of coalesce (#402) (#459) * Replace coalesce() usage for locals with ternary operator. Fixes terraform errors during destroy when only empty strings were passed to coalesce(). * Update changelog. * Fix formatting. --- CHANGELOG.md | 1 + local.tf | 25 ++++--------------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee4970c597..332fa94b82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed + - Fixed errors sometimes happening during destroy due to usage of coalesce() in local.tf (by @petrikero) - Write your awesome change here (by @you) # History diff --git a/local.tf b/local.tf index 915c8b274f..07c3504b4a 100644 --- a/local.tf +++ b/local.tf @@ -1,28 +1,11 @@ locals { asg_tags = null_resource.tags_as_list_of_maps.*.triggers - # Followed recommendation http://67bricks.com/blog/?p=85 - # to workaround terraform not supporting short circut evaluation - cluster_security_group_id = coalesce( - join("", aws_security_group.cluster.*.id), - var.cluster_security_group_id, - ) + cluster_security_group_id = var.cluster_create_security_group ? aws_security_group.cluster[0].id : var.cluster_security_group_id + cluster_iam_role_name = var.manage_cluster_iam_resources ? aws_iam_role.cluster[0].name : var.cluster_iam_role_name + cluster_iam_role_arn = var.manage_cluster_iam_resources ? aws_iam_role.cluster[0].arn : data.aws_iam_role.custom_cluster_iam_role[0].arn + worker_security_group_id = var.worker_create_security_group ? aws_security_group.workers[0].id : var.worker_security_group_id - cluster_iam_role_name = coalesce( - join("", aws_iam_role.cluster.*.name), - var.cluster_iam_role_name, - "aws-eks" - ) - cluster_iam_role_arn = coalesce( - join("", aws_iam_role.cluster.*.arn), - join("", data.aws_iam_role.custom_cluster_iam_role.*.arn), - "aws-eks" - ) - - worker_security_group_id = coalesce( - join("", aws_security_group.workers.*.id), - var.worker_security_group_id, - ) default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name From fb71eaf6ff2ae8599c5e0b64bff5f831183abeb3 Mon Sep 17 00:00:00 2001 From: Tarek Abdel Sater Date: Tue, 6 Aug 2019 19:18:06 +0300 Subject: [PATCH 042/962] Allow launch template spot instances without mixed policy (#463) * Add option to enable spot without MixedInstancePolicy * Update docs --- CHANGELOG.md | 1 + docs/spot-instances.md | 12 ++++++++++++ local.tf | 1 + workers_launch_template.tf | 8 ++++++++ 4 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 332fa94b82..7412f84011 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added + - Added `market_type` to `workers_launch_template.tf` allow the usage of spot nodegroups without mixed instances policy. - Added support for log group tag in `./cluster.tf` (@lucas-giaco) - Added support for workers iam role tag in `./workers.tf` (@lucas-giaco) - Write your awesome addition here (by @you) diff --git a/docs/spot-instances.md b/docs/spot-instances.md index ef9df35bc6..33bd807ff1 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -84,6 +84,18 @@ Launch Template support is a recent addition to both AWS and this module. It mig kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" } ] + + worker_groups_launch_template = [ + { + name = "spot-2" + instance_type = "m4.xlarge" + asg_max_size = 5 + asg_desired_size = 5 + autoscaling_enabled = true + kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" + market_type = "spot" + } + ] ``` ## Important issues diff --git a/local.tf b/local.tf index 07c3504b4a..ee1591bce6 100644 --- a/local.tf +++ b/local.tf @@ -56,6 +56,7 @@ locals { root_encrypted = "" # Whether the volume should be encrypted or not eni_delete = true # Delete the ENI on termination (if set to false you will have to manually delete before destroying) cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs + market_type = null # Settings for launch templates with mixed instances policy override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] # A list of override instance types for mixed instances policy on_demand_allocation_strategy = "prioritized" # Strategy to use when launching on-demand instances. Valid values: prioritized. diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 44bcd212c2..df4a231dd7 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -229,6 +229,14 @@ resource "aws_launch_template" "workers_launch_template" { ) } + dynamic instance_market_options { + iterator = item + for_each = lookup(var.worker_groups_launch_template[count.index], "market_type", null) == null ? [] : list(lookup(var.worker_groups_launch_template[count.index], "market_type", null)) + content { + market_type = item.value + } + } + block_device_mappings { device_name = lookup( var.worker_groups_launch_template[count.index], From b8b3b5820e18f6f054314e460ef5332c75d35e90 Mon Sep 17 00:00:00 2001 From: Karoline Pauls <43616133+karolinepauls@users.noreply.github.com> Date: Wed, 7 Aug 2019 08:43:09 +0100 Subject: [PATCH 043/962] basic example: correct elb tags (#458) Added the "kubernetes.io/role/elb" tag to the public subnets in the basic example. Per documentation, ELB tag values are supposed to be "1". ELB tag values being "true" are known not to work with aws-alb-ingress-controller. https://docs.aws.amazon.com/eks/latest/userguide/load-balancing.html https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html --- CHANGELOG.md | 1 + examples/basic/main.tf | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7412f84011..5d7d2be790 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Output the name of the cloudwatch log group (by @gbooth27) - Added `cpu_credits` param for the workers defined in `worker_groups_launch_template` (by @a-shink) - Added support for EBS Volumes tag in `worker_groups_launch_template` and `workers_launch_template_mixed.tf` (by @sppwf) + - Basic example now tags networks correctly, as per [ELB documentation](https://docs.aws.amazon.com/eks/latest/userguide/load-balancing.html) and [ALB documentation](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) (by @karolinepauls) ### Changed diff --git a/examples/basic/main.tf b/examples/basic/main.tf index c7d0cadc00..e68fd79c87 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -101,11 +101,12 @@ module "vpc" { public_subnet_tags = { "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/elb" = "1" } private_subnet_tags = { "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/internal-elb" = "true" + "kubernetes.io/role/internal-elb" = "1" } } From 8580b678139ff491db2ff25f414c913bf7d37008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=9B=B8=E8=BD=A9?= <9570112+nauxliu@users.noreply.github.com> Date: Mon, 19 Aug 2019 22:15:01 +0800 Subject: [PATCH 044/962] Support map users and roles to multiple groups (#424) * Support map users and roles to multiple groups * Simplify code by rename `user_arn` to `userarn`, `role_arn` to `rolearn` * Next version should be 6.x because PR this is a breaking change. * Update example variables.tf * Change indent to 2 * Fix map-aws-auth.yaml maybe invalid yaml. --- CHANGELOG.md | 3 +- aws_auth.tf | 44 ++----------------- examples/basic/variables.tf | 24 ++++++---- .../config-map-aws-auth-map_accounts.yaml.tpl | 1 - .../config-map-aws-auth-map_roles.yaml.tpl | 4 -- .../config-map-aws-auth-map_users.yaml.tpl | 4 -- templates/config-map-aws-auth.yaml.tpl | 12 +++-- variables.tf | 16 +++++-- 8 files changed, 42 insertions(+), 66 deletions(-) delete mode 100644 templates/config-map-aws-auth-map_accounts.yaml.tpl delete mode 100644 templates/config-map-aws-auth-map_roles.yaml.tpl delete mode 100644 templates/config-map-aws-auth-map_users.yaml.tpl diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d7d2be790..9f143071ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v5.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...HEAD)] - 2019-08-??] +## [[v6.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...HEAD)] - 2019-08-??] ### Added @@ -19,6 +19,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed + - Support map users and roles to multiple groups (by @nauxliu) - Fixed errors sometimes happening during destroy due to usage of coalesce() in local.tf (by @petrikero) - Write your awesome change here (by @you) diff --git a/aws_auth.tf b/aws_auth.tf index 6d3179a6b8..7d6930b957 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -95,46 +95,8 @@ data "template_file" "config_map_aws_auth" { ), ), ) - map_users = join("", data.template_file.map_users.*.rendered) - map_roles = join("", data.template_file.map_roles.*.rendered) - map_accounts = join("", data.template_file.map_accounts.*.rendered) + map_users = yamlencode(var.map_users), + map_roles = yamlencode(var.map_roles), + map_accounts = yamlencode(var.map_accounts) } } - -data "template_file" "map_users" { - count = length(var.map_users) - template = file( - "${path.module}/templates/config-map-aws-auth-map_users.yaml.tpl", - ) - - vars = { - user_arn = var.map_users[count.index]["user_arn"] - username = var.map_users[count.index]["username"] - group = var.map_users[count.index]["group"] - } -} - -data "template_file" "map_roles" { - count = length(var.map_roles) - template = file( - "${path.module}/templates/config-map-aws-auth-map_roles.yaml.tpl", - ) - - vars = { - role_arn = var.map_roles[count.index]["role_arn"] - username = var.map_roles[count.index]["username"] - group = var.map_roles[count.index]["group"] - } -} - -data "template_file" "map_accounts" { - count = length(var.map_accounts) - template = file( - "${path.module}/templates/config-map-aws-auth-map_accounts.yaml.tpl", - ) - - vars = { - account_number = var.map_accounts[count.index] - } -} - diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf index b3b96ac8d6..7085aeabd4 100644 --- a/examples/basic/variables.tf +++ b/examples/basic/variables.tf @@ -14,31 +14,39 @@ variable "map_accounts" { variable "map_roles" { description = "Additional IAM roles to add to the aws-auth configmap." - type = list(map(string)) + type = list(object({ + rolearn = string + username = string + groups = list(string) + })) default = [ { - role_arn = "arn:aws:iam::66666666666:role/role1" + rolearn = "arn:aws:iam::66666666666:role/role1" username = "role1" - group = "system:masters" + groups = ["system:masters"] }, ] } variable "map_users" { description = "Additional IAM users to add to the aws-auth configmap." - type = list(map(string)) + type = list(object({ + userarn = string + username = string + groups = list(string) + })) default = [ { - user_arn = "arn:aws:iam::66666666666:user/user1" + userarn = "arn:aws:iam::66666666666:user/user1" username = "user1" - group = "system:masters" + groups = ["system:masters"] }, { - user_arn = "arn:aws:iam::66666666666:user/user2" + userarn = "arn:aws:iam::66666666666:user/user2" username = "user2" - group = "system:masters" + groups = ["system:masters"] }, ] } diff --git a/templates/config-map-aws-auth-map_accounts.yaml.tpl b/templates/config-map-aws-auth-map_accounts.yaml.tpl deleted file mode 100644 index 26dc5078f4..0000000000 --- a/templates/config-map-aws-auth-map_accounts.yaml.tpl +++ /dev/null @@ -1 +0,0 @@ - - "${account_number}" diff --git a/templates/config-map-aws-auth-map_roles.yaml.tpl b/templates/config-map-aws-auth-map_roles.yaml.tpl deleted file mode 100644 index 9f321b7be6..0000000000 --- a/templates/config-map-aws-auth-map_roles.yaml.tpl +++ /dev/null @@ -1,4 +0,0 @@ - - rolearn: ${role_arn} - username: ${username} - groups: - - ${group} diff --git a/templates/config-map-aws-auth-map_users.yaml.tpl b/templates/config-map-aws-auth-map_users.yaml.tpl deleted file mode 100644 index 92499de41c..0000000000 --- a/templates/config-map-aws-auth-map_users.yaml.tpl +++ /dev/null @@ -1,4 +0,0 @@ - - userarn: ${user_arn} - username: ${username} - groups: - - ${group} diff --git a/templates/config-map-aws-auth.yaml.tpl b/templates/config-map-aws-auth.yaml.tpl index 86f4f5f998..1218956c17 100644 --- a/templates/config-map-aws-auth.yaml.tpl +++ b/templates/config-map-aws-auth.yaml.tpl @@ -6,8 +6,14 @@ metadata: data: mapRoles: | ${worker_role_arn} -${map_roles} + %{if chomp(map_roles) != "[]" } + ${indent(4, map_roles)} + %{ endif } + %{if chomp(map_users) != "[]" } mapUsers: | -${map_users} + ${indent(4, map_users)} + %{ endif } + %{if chomp(map_accounts) != "[]" } mapAccounts: | -${map_accounts} + ${indent(4, map_accounts)} + %{ endif } diff --git a/variables.tf b/variables.tf index 34b41e69df..9d391cc8e5 100644 --- a/variables.tf +++ b/variables.tf @@ -62,14 +62,22 @@ variable "map_accounts" { variable "map_roles" { description = "Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format." - type = list(map(string)) - default = [] + type = list(object({ + rolearn = string + username = string + groups = list(string) + })) + default = [] } variable "map_users" { description = "Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format." - type = list(map(string)) - default = [] + type = list(object({ + userarn = string + username = string + groups = list(string) + })) + default = [] } variable "subnets" { From b335819f0e1333755a1820e9db357ad7cd2022bf Mon Sep 17 00:00:00 2001 From: Scott Crooks Date: Mon, 19 Aug 2019 16:22:11 +0200 Subject: [PATCH 045/962] Adding 'capacity-optimized' docs to locals (#469) --- CHANGELOG.md | 2 +- local.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f143071ce..f877e4eea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added `market_type` to `workers_launch_template.tf` allow the usage of spot nodegroups without mixed instances policy. - Added support for log group tag in `./cluster.tf` (@lucas-giaco) - Added support for workers iam role tag in `./workers.tf` (@lucas-giaco) - - Write your awesome addition here (by @you) - Added `required_providers` to enforce provider minimum versions (by @dpiddockcmp) + - Updated `local.spot_allocation_strategy` docstring to indicate availability of new `capacity-optimized` option. (by @sc250024) ### Changed diff --git a/local.tf b/local.tf index ee1591bce6..d43118c95d 100644 --- a/local.tf +++ b/local.tf @@ -62,7 +62,7 @@ locals { on_demand_allocation_strategy = "prioritized" # Strategy to use when launching on-demand instances. Valid values: prioritized. on_demand_base_capacity = "0" # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances on_demand_percentage_above_base_capacity = "0" # Percentage split between on-demand and Spot instances above the base on-demand capacity - spot_allocation_strategy = "lowest-price" # The only valid value is lowest-price, which is also the default value. The Auto Scaling group selects the cheapest Spot pools and evenly allocates your Spot capacity across the number of Spot pools that you specify. + spot_allocation_strategy = "lowest-price" # Valid options are 'lowest-price' and 'capacity-optimized'. If 'lowest-price', the Auto Scaling group launches instances using the Spot pools with the lowest price, and evenly allocates your instances across the number of Spot pools. If 'capacity-optimized', the Auto Scaling group launches instances using Spot pools that are optimally chosen based on the available Spot capacity. spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price } From 655a75fbd0d551b7b49c4d9076d153a7219d902b Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Mon, 19 Aug 2019 16:34:02 +0200 Subject: [PATCH 046/962] Update cluster_security_group_id doc (#472) You cannot filter the public cluster endpoint. --- CHANGELOG.md | 1 + README.md | 2 +- variables.tf | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f877e4eea2..57f2de6a36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Support map users and roles to multiple groups (by @nauxliu) - Fixed errors sometimes happening during destroy due to usage of coalesce() in local.tf (by @petrikero) + - Removed historical mention of adding caller's IPv4 to cluster security group (by @dpiddockcmp) - Write your awesome change here (by @you) # History diff --git a/README.md b/README.md index 3c3e782774..1fdd43c7f6 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | string | `""` | no | | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | -| cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers and provide API access to your current IP/32. | string | `""` | no | +| cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers | string | `""` | no | | cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.13"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Should end in a forward slash `/` . | string | `"./"` | no | | iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | diff --git a/variables.tf b/variables.tf index 9d391cc8e5..5bbf1a9a89 100644 --- a/variables.tf +++ b/variables.tf @@ -20,7 +20,7 @@ variable "cluster_name" { } variable "cluster_security_group_id" { - description = "If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers and provide API access to your current IP/32." + description = "If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers" type = string default = "" } From 5636447de6a3059cf3a0586d3b486d53780c66ba Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Tue, 20 Aug 2019 09:41:16 -0400 Subject: [PATCH 047/962] Wrapping kubelet_extra_args in double quotes (#473) (#474) --- CHANGELOG.md | 1 + templates/userdata.sh.tpl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57f2de6a36..30bd304a3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fixed errors sometimes happening during destroy due to usage of coalesce() in local.tf (by @petrikero) - Removed historical mention of adding caller's IPv4 to cluster security group (by @dpiddockcmp) - Write your awesome change here (by @you) + - Wrapped `kubelet_extra_args` in double quotes instead of singe quotes (by @nxf5025) # History diff --git a/templates/userdata.sh.tpl b/templates/userdata.sh.tpl index ba8ea2800b..6194a863e5 100644 --- a/templates/userdata.sh.tpl +++ b/templates/userdata.sh.tpl @@ -4,7 +4,7 @@ ${pre_userdata} # Bootstrap and join the cluster -/etc/eks/bootstrap.sh --b64-cluster-ca '${cluster_auth_base64}' --apiserver-endpoint '${endpoint}' ${bootstrap_extra_args} --kubelet-extra-args '${kubelet_extra_args}' '${cluster_name}' +/etc/eks/bootstrap.sh --b64-cluster-ca '${cluster_auth_base64}' --apiserver-endpoint '${endpoint}' ${bootstrap_extra_args} --kubelet-extra-args "${kubelet_extra_args}" '${cluster_name}' # Allow user supplied userdata code ${additional_userdata} From d8ed7d0b669a9d7ca1741c094c70ed00267bc661 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 20 Aug 2019 15:43:18 +0200 Subject: [PATCH 048/962] add option to recreate ASG when LT or LC changes (#465) --- CHANGELOG.md | 1 + local.tf | 1 + versions.tf | 1 + workers.tf | 33 +++++++++++++++++++++++++++--- workers_launch_template.tf | 34 ++++++++++++++++++++++++++----- workers_launch_template_mixed.tf | 35 +++++++++++++++++++++++++++----- 6 files changed, 92 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30bd304a3d..2998cce6fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added support for workers iam role tag in `./workers.tf` (@lucas-giaco) - Added `required_providers` to enforce provider minimum versions (by @dpiddockcmp) - Updated `local.spot_allocation_strategy` docstring to indicate availability of new `capacity-optimized` option. (by @sc250024) + - Added option to recreate ASG when LT or LC changes (by @barryib) ### Changed diff --git a/local.tf b/local.tf index d43118c95d..98d3909494 100644 --- a/local.tf +++ b/local.tf @@ -21,6 +21,7 @@ locals { asg_max_size = "3" # Maximum worker capacity in the autoscaling group. asg_min_size = "1" # Minimum worker capacity in the autoscaling group. asg_force_delete = false # Enable forced deletion for the autoscaling group. + asg_recreate_on_change = false # Recreate the autoscaling group when LT or LC change. instance_type = "m4.large" # Size of the workers instances. spot_price = "" # Cost of spot instance. placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". diff --git a/versions.tf b/versions.tf index affd035db0..aff0bf8375 100644 --- a/versions.tf +++ b/versions.tf @@ -6,5 +6,6 @@ terraform { local = ">= 1.2" null = ">= 2.1" template = ">= 2.1" + random = ">= 2.1" } } diff --git a/workers.tf b/workers.tf index 8a330d4a3f..7446855649 100644 --- a/workers.tf +++ b/workers.tf @@ -1,8 +1,17 @@ # Worker Groups using Launch Configurations resource "aws_autoscaling_group" "workers" { - count = local.worker_group_count - name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}" + count = local.worker_group_count + name_prefix = join( + "-", + compact( + [ + aws_eks_cluster.this.name, + lookup(var.worker_groups[count.index], "name", count.index), + lookup(var.worker_groups[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers[count.index].id : "" + ] + ) + ) desired_capacity = lookup( var.worker_groups[count.index], "asg_desired_capacity", @@ -210,6 +219,25 @@ resource "aws_launch_configuration" "workers" { } } +resource "random_pet" "workers" { + count = local.worker_group_count + + separator = "-" + length = 2 + + keepers = { + lt_name = join( + "-", + compact( + [ + aws_launch_configuration.workers[count.index].name, + aws_launch_configuration.workers[count.index].latest_version + ] + ) + ) + } +} + resource "aws_security_group" "workers" { count = var.worker_create_security_group ? 1 : 0 name_prefix = aws_eks_cluster.this.name @@ -390,4 +418,3 @@ data "aws_iam_policy_document" "worker_autoscaling" { } } } - diff --git a/workers_launch_template.tf b/workers_launch_template.tf index df4a231dd7..46eef4c2f0 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -2,11 +2,16 @@ resource "aws_autoscaling_group" "workers_launch_template" { count = local.worker_group_launch_template_count - name_prefix = "${aws_eks_cluster.this.name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}" + name_prefix = join( + "-", + compact( + [ + aws_eks_cluster.this.name, + lookup(var.worker_groups_launch_template[count.index], "name", count.index), + lookup(var.worker_groups_launch_template[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers_launch_template[count.index].id : "" + ] + ) + ) desired_capacity = lookup( var.worker_groups_launch_template[count.index], "asg_desired_capacity", @@ -296,6 +301,25 @@ resource "aws_launch_template" "workers_launch_template" { } } +resource "random_pet" "workers_launch_template" { + count = local.worker_group_launch_template_count + + separator = "-" + length = 2 + + keepers = { + lt_name = join( + "-", + compact( + [ + aws_launch_template.workers_launch_template[count.index].name, + aws_launch_template.workers_launch_template[count.index].latest_version + ] + ) + ) + } +} + resource "aws_iam_instance_profile" "workers_launch_template" { count = var.manage_worker_iam_resources ? local.worker_group_launch_template_count : 0 name_prefix = aws_eks_cluster.this.name diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf index 61df13670d..75489e990f 100644 --- a/workers_launch_template_mixed.tf +++ b/workers_launch_template_mixed.tf @@ -2,11 +2,17 @@ resource "aws_autoscaling_group" "workers_launch_template_mixed" { count = local.worker_group_launch_template_mixed_count - name_prefix = "${aws_eks_cluster.this.name}-${lookup( - var.worker_groups_launch_template_mixed[count.index], - "name", - count.index, - )}" + name_prefix = join( + "-", + compact( + [ + aws_eks_cluster.this.name, + lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index), + lookup(var.worker_groups_launch_template_mixed[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers_launch_template_mixed[count.index].id : "" + ] + ) + ) + desired_capacity = lookup( var.worker_groups_launch_template_mixed[count.index], "asg_desired_capacity", @@ -338,6 +344,25 @@ resource "aws_launch_template" "workers_launch_template_mixed" { } } +resource "random_pet" "workers_launch_template_mixed" { + count = local.worker_group_launch_template_mixed_count + + separator = "-" + length = 2 + + keepers = { + lt_name = join( + "-", + compact( + [ + aws_launch_template.workers_launch_template_mixed[count.index].name, + aws_launch_template.workers_launch_template_mixed[count.index].latest_version + ] + ) + ) + } +} + resource "aws_iam_instance_profile" "workers_launch_template_mixed" { count = var.manage_worker_iam_resources ? local.worker_group_launch_template_mixed_count : 0 name_prefix = aws_eks_cluster.this.name From d6d097545278f07e18261127d1fbc51feb43bb93 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 20 Aug 2019 15:45:06 +0200 Subject: [PATCH 049/962] add initial lifecycle hooks for autosacling groups (#466) --- CHANGELOG.md | 1 + local.tf | 1 + workers.tf | 13 +++++++++++++ workers_launch_template.tf | 13 +++++++++++++ workers_launch_template_mixed.tf | 13 +++++++++++++ 5 files changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2998cce6fa..f211f88ab5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added support for workers iam role tag in `./workers.tf` (@lucas-giaco) - Added `required_providers` to enforce provider minimum versions (by @dpiddockcmp) - Updated `local.spot_allocation_strategy` docstring to indicate availability of new `capacity-optimized` option. (by @sc250024) + - Added support for initial lifecycle hooks for autosacling groups (@barryib) - Added option to recreate ASG when LT or LC changes (by @barryib) ### Changed diff --git a/local.tf b/local.tf index 98d3909494..c4cdab0a14 100644 --- a/local.tf +++ b/local.tf @@ -21,6 +21,7 @@ locals { asg_max_size = "3" # Maximum worker capacity in the autoscaling group. asg_min_size = "1" # Minimum worker capacity in the autoscaling group. asg_force_delete = false # Enable forced deletion for the autoscaling group. + asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. asg_recreate_on_change = false # Recreate the autoscaling group when LT or LC change. instance_type = "m4.large" # Size of the workers instances. spot_price = "" # Cost of spot instance. diff --git a/workers.tf b/workers.tf index 7446855649..d924308809 100644 --- a/workers.tf +++ b/workers.tf @@ -74,6 +74,19 @@ resource "aws_autoscaling_group" "workers" { local.workers_group_defaults["termination_policies"] ) + dynamic "initial_lifecycle_hook" { + for_each = lookup(var.worker_groups[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) + content { + name = lookup(initial_lifecycle_hook.value, "name", null) + lifecycle_transition = lookup(initial_lifecycle_hook.value, "lifecycle_transition", null) + notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) + heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) + notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) + role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) + default_result = lookup(initial_lifecycle_hook.value, "default_result", null) + } + } + tags = concat( [ { diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 46eef4c2f0..18d09ba92d 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -82,6 +82,19 @@ resource "aws_autoscaling_group" "workers_launch_template" { ) } + dynamic "initial_lifecycle_hook" { + for_each = lookup(var.worker_groups_launch_template[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) + content { + name = lookup(initial_lifecycle_hook.value, "name", null) + lifecycle_transition = lookup(initial_lifecycle_hook.value, "lifecycle_transition", null) + notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) + heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) + notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) + role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) + default_result = lookup(initial_lifecycle_hook.value, "default_result", null) + } + } + tags = concat( [ { diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf index 75489e990f..5bf66c499c 100644 --- a/workers_launch_template_mixed.tf +++ b/workers_launch_template_mixed.tf @@ -133,6 +133,19 @@ resource "aws_autoscaling_group" "workers_launch_template_mixed" { } } + dynamic "initial_lifecycle_hook" { + for_each = lookup(var.worker_groups_launch_template_mixed[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) + content { + name = lookup(initial_lifecycle_hook.value, "name", null) + default_result = lookup(initial_lifecycle_hook.value, "default_result", null) + heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) + lifecycle_transition = lookup(initial_lifecycle_hook.value, "lifecycle_transition", null) + notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) + notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) + role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) + } + } + tags = concat( [ { From 97df53c0994d62f924133e2b5fd0bb67e5f7b11b Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 27 Aug 2019 17:49:21 +0200 Subject: [PATCH 050/962] launch configuration doen't have version (#482) close #479 --- workers.tf | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/workers.tf b/workers.tf index d924308809..cda6865090 100644 --- a/workers.tf +++ b/workers.tf @@ -239,15 +239,7 @@ resource "random_pet" "workers" { length = 2 keepers = { - lt_name = join( - "-", - compact( - [ - aws_launch_configuration.workers[count.index].name, - aws_launch_configuration.workers[count.index].latest_version - ] - ) - ) + lc_name = aws_launch_configuration.workers[count.index].name } } From 6c3e4ec510f658f53508623a6192df064e7a4786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=9B=B8=E8=BD=A9?= <9570112+nauxliu@users.noreply.github.com> Date: Mon, 9 Sep 2019 16:46:02 +0800 Subject: [PATCH 051/962] Update to EKS 1.14 (#498) --- CHANGELOG.md | 3 ++- README.md | 2 +- variables.tf | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f211f88ab5..fd848f07d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,11 +21,12 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed + - Update to EKS 1.14 (by @nauxliu) - Support map users and roles to multiple groups (by @nauxliu) - Fixed errors sometimes happening during destroy due to usage of coalesce() in local.tf (by @petrikero) - Removed historical mention of adding caller's IPv4 to cluster security group (by @dpiddockcmp) - - Write your awesome change here (by @you) - Wrapped `kubelet_extra_args` in double quotes instead of singe quotes (by @nxf5025) + - Write your awesome change here (by @you) # History diff --git a/README.md b/README.md index 1fdd43c7f6..e302bb9a23 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers | string | `""` | no | -| cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.13"` | no | +| cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.14"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Should end in a forward slash `/` . | string | `"./"` | no | | iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | | kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | diff --git a/variables.tf b/variables.tf index 5bbf1a9a89..c85ac17b27 100644 --- a/variables.tf +++ b/variables.tf @@ -28,7 +28,7 @@ variable "cluster_security_group_id" { variable "cluster_version" { description = "Kubernetes version to use for the EKS cluster." type = string - default = "1.13" + default = "1.14" } variable "config_output_path" { From 2254ab709e3f6b340d4962735539a9560c661b28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=9B=B8=E8=BD=A9?= <9570112+nauxliu@users.noreply.github.com> Date: Wed, 11 Sep 2019 22:59:58 +0800 Subject: [PATCH 052/962] Replace travis with GitHub Actions (#497) --- .github/workflows/lint.yml | 41 ++++++++++++++++++++++++++++++ .ruby-version | 2 +- .travis.yml | 52 -------------------------------------- Gemfile | 2 +- 4 files changed, 43 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/lint.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..0cafaedd9f --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,41 @@ +name: Lint +on: [push, pull_request] + +jobs: + validate: + name: Validate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + # - uses: actions/setup-ruby@v1 + # with: + # ruby-version: '2.4.6' + # - name: Install Kitchen-Terraform + # run: | + # gem install bundler --no-document + # bundle install + - name: Install Terraform + run: | + export TERRAFORM_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version') + curl --silent --output terraform.zip "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" + unzip terraform.zip ; rm -f terraform.zip; chmod +x terraform + mkdir -p ${HOME}/bin ; export PATH=${PATH}:${HOME}/bin; mv terraform ${HOME}/bin/ + terraform -v + - name: Validate Code + env: + AWS_REGION: 'us-east-1' + TF_VAR_region: ${AWS_REGION} + TF_WARN_OUTPUT_ERRORS: 1 + run: | + terraform init + terraform fmt -check=true + terraform validate -var "region=${AWS_REGION}" -var "vpc_id=vpc-123456" -var "subnets=[\"subnet-12345a\"]" -var "workers_ami_id=ami-123456" -var "cluster_ingress_cidrs=[]" -var "cluster_name=test_cluster" + - name: Validate Examples + run: | + for example in $(find examples -maxdepth 1 -mindepth 1 -type d); do + cd $example + terraform init + terraform fmt -check=true + terraform validate + cd - + done diff --git a/.ruby-version b/.ruby-version index 79a614418f..7bf4b6a8ae 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.4.4 +2.4.6 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7954f76975..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,52 +0,0 @@ -language: ruby -sudo: required -dist: trusty - -services: -- docker - -rvm: -- 2.4.4 - -env: - global: - - AWS_REGION='us-east-1' - - TF_VAR_region=${AWS_REGION} - - TF_WARN_OUTPUT_ERRORS=1 - -before_install: -- echo "before_install" - -install: -- echo "install" -- gem install bundler --no-rdoc --no-ri -- bundle install - -before_script: -- export TERRAFORM_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version') -- curl --silent --output terraform.zip "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -- unzip terraform.zip ; rm -f terraform.zip; chmod +x terraform -- mkdir -p ${HOME}/bin ; export PATH=${PATH}:${HOME}/bin; mv terraform ${HOME}/bin/ -- terraform -v - -script: -- echo 'script' -- terraform init -- terraform fmt -check=true -- terraform validate -var "region=${AWS_REGION}" -var "vpc_id=vpc-123456" -var "subnets=[\"subnet-12345a\"]" -var "workers_ami_id=ami-123456" -var "cluster_ingress_cidrs=[]" -var "cluster_name=test_cluster" -- | - for example in $(find examples -maxdepth 1 -mindepth 1 -type d); do - cd $example - terraform init - terraform fmt -check=true - terraform validate - cd - - done -- terraform -v - -notifications: - email: - recipients: - - brandon@atscale.run - on_success: change - on_failure: change diff --git a/Gemfile b/Gemfile index 89be253ef9..57a0c15c31 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ # frozen_string_literal: true -ruby '2.4.4' +ruby '2.4.6' source 'https://rubygems.org/' do gem 'awspec', '~> 1.4.2' From 4264a5fe6ef6ca157644221fefa944e24a0ccee5 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 11 Sep 2019 18:25:33 +0200 Subject: [PATCH 053/962] use default_iam_role_id interpolation for aws_iam_instance_profile (#489) --- CHANGELOG.md | 1 + local.tf | 70 ++++++++++++++++---------------- workers.tf | 2 +- workers_launch_template.tf | 2 +- workers_launch_template_mixed.tf | 2 +- 5 files changed, 39 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd848f07d2..55ce4a23e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Removed historical mention of adding caller's IPv4 to cluster security group (by @dpiddockcmp) - Wrapped `kubelet_extra_args` in double quotes instead of singe quotes (by @nxf5025) - Write your awesome change here (by @you) + - Make terraform plan more consistent and avoid unnecessary "(known after apply)" (by @barryib) # History diff --git a/local.tf b/local.tf index c4cdab0a14..a66452f3b8 100644 --- a/local.tf +++ b/local.tf @@ -14,41 +14,41 @@ locals { worker_group_launch_template_mixed_count = length(var.worker_groups_launch_template_mixed) workers_group_defaults_defaults = { - name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. - tags = [] # A list of map defining extra tags to be applied to the worker group ASG. - ami_id = data.aws_ami.eks_worker.id # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI. - asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group. - asg_max_size = "3" # Maximum worker capacity in the autoscaling group. - asg_min_size = "1" # Minimum worker capacity in the autoscaling group. - asg_force_delete = false # Enable forced deletion for the autoscaling group. - asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. - asg_recreate_on_change = false # Recreate the autoscaling group when LT or LC change. - instance_type = "m4.large" # Size of the workers instances. - spot_price = "" # Cost of spot instance. - placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". - root_volume_size = "100" # root volume size of workers instances. - root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp2', or 'io1' - root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". - key_name = "" # The key name that should be used for the instances in the autoscaling group - pre_userdata = "" # userdata to pre-append to the default userdata. - bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI. - additional_userdata = "" # userdata to append to the default userdata. - ebs_optimized = true # sets whether to use ebs optimization on supported types. - enable_monitoring = true # Enables/disables detailed monitoring. - public_ip = false # Associate a public ip address with a worker - kubelet_extra_args = "" # This string is passed directly to kubelet if set. Useful for adding labels or taints. - subnets = var.subnets # A list of subnets to place the worker nodes in. i.e. ["subnet-123", "subnet-456", "subnet-789"] - autoscaling_enabled = false # Sets whether policy and matching tags will be added to allow autoscaling. - additional_security_group_ids = [] # A list of additional security group ids to include in worker launch config - protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. - iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. - iam_role_id = local.default_iam_role_id # A custom IAM role id. Incompatible with iam_instance_profile_name. - suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] - target_group_arns = [] # A list of ALB target group ARNs to be associated to the ASG - enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] - placement_group = "" # The name of the placement group into which to launch the instances, if any. - service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS - termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. + name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. + tags = [] # A list of map defining extra tags to be applied to the worker group ASG. + ami_id = data.aws_ami.eks_worker.id # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI. + asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group. + asg_max_size = "3" # Maximum worker capacity in the autoscaling group. + asg_min_size = "1" # Minimum worker capacity in the autoscaling group. + asg_force_delete = false # Enable forced deletion for the autoscaling group. + asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. + asg_recreate_on_change = false # Recreate the autoscaling group when LT or LC change. + instance_type = "m4.large" # Size of the workers instances. + spot_price = "" # Cost of spot instance. + placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". + root_volume_size = "100" # root volume size of workers instances. + root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp2', or 'io1' + root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". + key_name = "" # The key name that should be used for the instances in the autoscaling group + pre_userdata = "" # userdata to pre-append to the default userdata. + bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI. + additional_userdata = "" # userdata to append to the default userdata. + ebs_optimized = true # sets whether to use ebs optimization on supported types. + enable_monitoring = true # Enables/disables detailed monitoring. + public_ip = false # Associate a public ip address with a worker + kubelet_extra_args = "" # This string is passed directly to kubelet if set. Useful for adding labels or taints. + subnets = var.subnets # A list of subnets to place the worker nodes in. i.e. ["subnet-123", "subnet-456", "subnet-789"] + autoscaling_enabled = false # Sets whether policy and matching tags will be added to allow autoscaling. + additional_security_group_ids = [] # A list of additional security group ids to include in worker launch config + protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. + iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. + iam_role_id = "local.default_iam_role_id" # A custom IAM role id. Incompatible with iam_instance_profile_name. Literal local.default_iam_role_id will never be used but if iam_role_id is not set, the local.default_iam_role_id interpolation will be used. + suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] + target_group_arns = [] # A list of ALB target group ARNs to be associated to the ASG + enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] + placement_group = "" # The name of the placement group into which to launch the instances, if any. + service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS + termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device diff --git a/workers.tf b/workers.tf index cda6865090..ec919024d2 100644 --- a/workers.tf +++ b/workers.tf @@ -328,7 +328,7 @@ resource "aws_iam_instance_profile" "workers" { role = lookup( var.worker_groups[count.index], "iam_role_id", - local.workers_group_defaults["iam_role_id"], + local.default_iam_role_id, ) path = var.iam_path diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 18d09ba92d..6860187651 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -339,7 +339,7 @@ resource "aws_iam_instance_profile" "workers_launch_template" { role = lookup( var.worker_groups_launch_template[count.index], "iam_role_id", - local.workers_group_defaults["iam_role_id"], + local.default_iam_role_id, ) path = var.iam_path } diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf index 5bf66c499c..3faa8a7ae8 100644 --- a/workers_launch_template_mixed.tf +++ b/workers_launch_template_mixed.tf @@ -382,7 +382,7 @@ resource "aws_iam_instance_profile" "workers_launch_template_mixed" { role = lookup( var.worker_groups_launch_template_mixed[count.index], "iam_role_id", - local.workers_group_defaults["iam_role_id"], + local.default_iam_role_id, ) path = var.iam_path } From a47f4642215734ce249ad0f9bb559bafe2f03c8e Mon Sep 17 00:00:00 2001 From: Ivanich Date: Thu, 12 Sep 2019 18:45:04 +0300 Subject: [PATCH 054/962] Allow to set user defined workers role name (#496) * Allow to set user defined workers role name * Add changelog entry * Keep single variable for worker role name --- CHANGELOG.md | 1 + README.md | 1 + variables.tf | 6 ++++++ workers.tf | 3 ++- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55ce4a23e8..0dc6d83630 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Updated `local.spot_allocation_strategy` docstring to indicate availability of new `capacity-optimized` option. (by @sc250024) - Added support for initial lifecycle hooks for autosacling groups (@barryib) - Added option to recreate ASG when LT or LC changes (by @barryib) + - Ability to specify workers role name (by @ivanich) ### Changed diff --git a/README.md b/README.md index e302bb9a23..b021517b0a 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | manage\_aws\_auth | Whether to apply the aws-auth configmap file. | string | `"true"` | no | | manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | bool | `"true"` | no | | manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | bool | `"true"` | no | +| workers\_role\_name | User defined workers role name. | sting | `""` | no | | map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | | map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(map(string)) | `[]` | no | | map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(map(string)) | `[]` | no | diff --git a/variables.tf b/variables.tf index c85ac17b27..1a1bf0dbea 100644 --- a/variables.tf +++ b/variables.tf @@ -251,3 +251,9 @@ variable "manage_worker_iam_resources" { type = bool default = true } + +variable "workers_role_name" { + description = "User defined workers role name." + type = string + default = "" +} diff --git a/workers.tf b/workers.tf index ec919024d2..432d1bbc19 100644 --- a/workers.tf +++ b/workers.tf @@ -314,7 +314,8 @@ resource "aws_security_group_rule" "workers_ingress_cluster_https" { resource "aws_iam_role" "workers" { count = var.manage_worker_iam_resources ? 1 : 0 - name_prefix = aws_eks_cluster.this.name + name_prefix = var.workers_role_name != "" ? null : aws_eks_cluster.this.name + name = var.workers_role_name != "" ? var.workers_role_name : null assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json permissions_boundary = var.permissions_boundary path = var.iam_path From 461cf5482ebde7fc9d5604765e91547368763fc5 Mon Sep 17 00:00:00 2001 From: Sergiu Plotnicu <33776238+sppwf@users.noreply.github.com> Date: Fri, 13 Sep 2019 17:50:59 +0300 Subject: [PATCH 055/962] Support for Mixed Instances ASG in worker_groups_launch_template variable (#468) * Create ASG tags via for - utility from terraform 12 * Updated support for mixed ASG in worker_groups_launch_template variable * Updated launch_template example to include spot and mixed ASG with worker_groups_launch_template variable * Removed old config * Removed workers_launch_template_mixed.tf file, added support for mixed/spot in workers_launch_template variable * Updated examples/spot_instances/main.tf with Mixed Spot and ondemand instances * Removed launch_template_mixed from relevant files * Updated README.md file * Removed workers_launch_template.tf.bkp * Fixed case with null on_demand_allocation_strategy and Spot allocation * Fixed workers_launch_template.tf, covered spot instances via Launch Template --- CHANGELOG.md | 3 + README.md | 13 +- aws_auth.tf | 16 - data.tf | 41 --- examples/launch_templates/pre_userdata.sh | 1 + examples/spot_instances/main.tf | 2 +- local.tf | 16 +- outputs.tf | 4 - variables.tf | 6 - workers.tf | 10 - workers_launch_template.tf | 82 ++++- workers_launch_template_mixed.tf | 388 ---------------------- 12 files changed, 97 insertions(+), 485 deletions(-) create mode 100644 examples/launch_templates/pre_userdata.sh delete mode 100644 workers_launch_template_mixed.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dc6d83630..7f503c45e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added support for initial lifecycle hooks for autosacling groups (@barryib) - Added option to recreate ASG when LT or LC changes (by @barryib) - Ability to specify workers role name (by @ivanich) + - Added support for Mixed Instance ASG using `worker_groups_launch_template` variable (by @sppwf) + - Changed ASG Tags generation using terraform 12 `for` utility (by @sppwf) + - Removed `worker_groups_launch_template_mixed` variable (by @sppwf) ### Changed diff --git a/README.md b/README.md index b021517b0a..f567503c6b 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | list(string) | `[]` | no | | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | bool | `"false"` | no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | bool | `"true"` | no | -| cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. | string | `""` | no | +| cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. | string | `""` | no | | cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | string | `""` | no | | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | @@ -126,7 +126,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.14"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Should end in a forward slash `/` . | string | `"./"` | no | | iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | -| kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | +| kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. \["-r", "MyEksRole"\]. | list(string) | `[]` | no | | kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | string | `"aws-iam-authenticator"` | no | | kubeconfig\_aws\_authenticator\_command\_args | Default arguments passed to the authenticator command. Defaults to [token -i $cluster_name]. | list(string) | `[]` | no | | kubeconfig\_aws\_authenticator\_env\_variables | Environment variables that should be used when executing the authenticator. e.g. { AWS_PROFILE = "eks"}. | map(string) | `{}` | no | @@ -144,15 +144,14 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | tags | A map of tags to add to all resources. | map(string) | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | -| worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | +| worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster\_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | -| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | -| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | -| worker\_groups\_launch\_template\_mixed | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | +| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers\_group\_defaults for valid keys. | any | `[]` | no | +| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | any | `[]` | no | | worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster. | string | `""` | no | | worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | number | `"1025"` | no | | workers\_additional\_policies | Additional policies to be added to workers | list(string) | `[]` | no | -| workers\_group\_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | any | `{}` | no | +| workers\_group\_defaults | Override default values for target groups. See workers\_group\_defaults\_defaults in local.tf for valid keys. | any | `{}` | no | | write\_aws\_auth\_config | Whether to write the aws-auth configmap file. | bool | `"true"` | no | | write\_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | bool | `"true"` | no | diff --git a/aws_auth.tf b/aws_auth.tf index 7d6930b957..d6531107cb 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -35,21 +35,6 @@ EOS data "aws_caller_identity" "current" { } -data "template_file" "launch_template_mixed_worker_role_arns" { - count = local.worker_group_launch_template_mixed_count - template = file("${path.module}/templates/worker-role.tpl") - - vars = { - worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element( - coalescelist( - aws_iam_instance_profile.workers_launch_template_mixed.*.role, - data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.role_name, - ), - count.index, - )}" - } -} - data "template_file" "launch_template_worker_role_arns" { count = local.worker_group_launch_template_count template = file("${path.module}/templates/worker-role.tpl") @@ -91,7 +76,6 @@ data "template_file" "config_map_aws_auth" { concat( data.template_file.launch_template_worker_role_arns.*.rendered, data.template_file.worker_role_arns.*.rendered, - data.template_file.launch_template_mixed_worker_role_arns.*.rendered, ), ), ) diff --git a/data.tf b/data.tf index 01b1145fad..c1f92ca4e1 100644 --- a/data.tf +++ b/data.tf @@ -147,37 +147,6 @@ data "template_file" "launch_template_userdata" { } } -data "template_file" "workers_launch_template_mixed" { - count = local.worker_group_launch_template_mixed_count - template = file("${path.module}/templates/userdata.sh.tpl") - - vars = { - cluster_name = aws_eks_cluster.this.name - endpoint = aws_eks_cluster.this.endpoint - cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data - pre_userdata = lookup( - var.worker_groups_launch_template_mixed[count.index], - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - var.worker_groups_launch_template_mixed[count.index], - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - var.worker_groups_launch_template_mixed[count.index], - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - var.worker_groups_launch_template_mixed[count.index], - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - } -} - data "aws_iam_role" "custom_cluster_iam_role" { count = var.manage_cluster_iam_resources ? 0 : 1 name = var.cluster_iam_role_name @@ -200,13 +169,3 @@ data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instanc local.workers_group_defaults["iam_instance_profile_name"], ) } - -data "aws_iam_instance_profile" "custom_worker_group_launch_template_mixed_iam_instance_profile" { - count = var.manage_worker_iam_resources ? 0 : local.worker_group_launch_template_mixed_count - name = lookup( - var.worker_groups_launch_template_mixed[count.index], - "iam_instance_profile_name", - local.workers_group_defaults["iam_instance_profile_name"], - ) -} - diff --git a/examples/launch_templates/pre_userdata.sh b/examples/launch_templates/pre_userdata.sh new file mode 100644 index 0000000000..52dd50f28c --- /dev/null +++ b/examples/launch_templates/pre_userdata.sh @@ -0,0 +1 @@ +yum update -y diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index c8550e21d2..0a73642cee 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -56,7 +56,7 @@ module "eks" { subnets = module.vpc.public_subnets vpc_id = module.vpc.vpc_id - worker_groups_launch_template_mixed = [ + worker_groups_launch_template = [ { name = "spot-1" override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] diff --git a/local.tf b/local.tf index a66452f3b8..e5a9c33b6c 100644 --- a/local.tf +++ b/local.tf @@ -1,5 +1,12 @@ locals { - asg_tags = null_resource.tags_as_list_of_maps.*.triggers + asg_tags = [ + for item in keys(var.tags) : + map( + "key", item, + "value", element(values(var.tags), index(keys(var.tags), item)), + "propagate_at_launch", "true" + ) + ] cluster_security_group_id = var.cluster_create_security_group ? aws_security_group.cluster[0].id : var.cluster_security_group_id cluster_iam_role_name = var.manage_cluster_iam_resources ? aws_iam_role.cluster[0].name : var.cluster_iam_role_name @@ -9,9 +16,8 @@ locals { default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name - worker_group_count = length(var.worker_groups) - worker_group_launch_template_count = length(var.worker_groups_launch_template) - worker_group_launch_template_mixed_count = length(var.worker_groups_launch_template_mixed) + worker_group_count = length(var.worker_groups) + worker_group_launch_template_count = length(var.worker_groups_launch_template) workers_group_defaults_defaults = { name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. @@ -61,7 +67,7 @@ locals { market_type = null # Settings for launch templates with mixed instances policy override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] # A list of override instance types for mixed instances policy - on_demand_allocation_strategy = "prioritized" # Strategy to use when launching on-demand instances. Valid values: prioritized. + on_demand_allocation_strategy = null # Strategy to use when launching on-demand instances. Valid values: prioritized. on_demand_base_capacity = "0" # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances on_demand_percentage_above_base_capacity = "0" # Percentage split between on-demand and Spot instances above the base on-demand capacity spot_allocation_strategy = "lowest-price" # Valid options are 'lowest-price' and 'capacity-optimized'. If 'lowest-price', the Auto Scaling group launches instances using the Spot pools with the lowest price, and evenly allocates your instances across the number of Spot pools. If 'capacity-optimized', the Auto Scaling group launches instances using Spot pools that are optimally chosen based on the available Spot capacity. diff --git a/outputs.tf b/outputs.tf index a65c76fe4a..8f48268f07 100644 --- a/outputs.tf +++ b/outputs.tf @@ -63,7 +63,6 @@ output "workers_asg_arns" { value = concat( aws_autoscaling_group.workers.*.arn, aws_autoscaling_group.workers_launch_template.*.arn, - aws_autoscaling_group.workers_launch_template_mixed.*.arn, ) } @@ -72,7 +71,6 @@ output "workers_asg_names" { value = concat( aws_autoscaling_group.workers.*.id, aws_autoscaling_group.workers_launch_template.*.id, - aws_autoscaling_group.workers_launch_template_mixed.*.id, ) } @@ -125,7 +123,6 @@ output "worker_iam_role_name" { aws_iam_role.workers.*.name, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, - data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.role_name, [""] )[0] } @@ -136,7 +133,6 @@ output "worker_iam_role_arn" { aws_iam_role.workers.*.arn, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_arn, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_arn, - data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.role_arn, [""] )[0] } diff --git a/variables.tf b/variables.tf index 1a1bf0dbea..bb8a7ffd0b 100644 --- a/variables.tf +++ b/variables.tf @@ -114,12 +114,6 @@ variable "worker_groups_launch_template" { default = [] } -variable "worker_groups_launch_template_mixed" { - description = "A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys." - type = any - default = [] -} - variable "worker_security_group_id" { description = "If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster." type = string diff --git a/workers.tf b/workers.tf index 432d1bbc19..45db1e449a 100644 --- a/workers.tf +++ b/workers.tf @@ -359,16 +359,6 @@ resource "aws_iam_role_policy_attachment" "workers_additional_policies" { policy_arn = var.workers_additional_policies[count.index] } -resource "null_resource" "tags_as_list_of_maps" { - count = length(keys(var.tags)) - - triggers = { - key = keys(var.tags)[count.index] - value = values(var.tags)[count.index] - propagate_at_launch = "true" - } -} - resource "aws_iam_role_policy_attachment" "workers_autoscaling" { count = var.manage_worker_iam_resources ? 1 : 0 policy_arn = aws_iam_policy.worker_autoscaling[0].arn diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 6860187651..ce1c21d4a7 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -73,13 +73,81 @@ resource "aws_autoscaling_group" "workers_launch_template" { local.workers_group_defaults["termination_policies"] ) - launch_template { - id = aws_launch_template.workers_launch_template.*.id[count.index] - version = lookup( - var.worker_groups_launch_template[count.index], - "launch_template_version", - local.workers_group_defaults["launch_template_version"], - ) + dynamic mixed_instances_policy { + iterator = item + for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", null) != null) ? list(var.worker_groups_launch_template[count.index]) : [] + + content { + instances_distribution { + on_demand_allocation_strategy = lookup( + item.value, + "on_demand_allocation_strategy", + "prioritized", + ) + on_demand_base_capacity = lookup( + item.value, + "on_demand_base_capacity", + local.workers_group_defaults["on_demand_base_capacity"], + ) + on_demand_percentage_above_base_capacity = lookup( + item.value, + "on_demand_percentage_above_base_capacity", + local.workers_group_defaults["on_demand_percentage_above_base_capacity"], + ) + spot_allocation_strategy = lookup( + item.value, + "spot_allocation_strategy", + local.workers_group_defaults["spot_allocation_strategy"], + ) + spot_instance_pools = lookup( + item.value, + "spot_instance_pools", + local.workers_group_defaults["spot_instance_pools"], + ) + spot_max_price = lookup( + item.value, + "spot_max_price", + local.workers_group_defaults["spot_max_price"], + ) + } + + launch_template { + launch_template_specification { + launch_template_id = aws_launch_template.workers_launch_template.*.id[count.index] + version = lookup( + var.worker_groups_launch_template[count.index], + "launch_template_version", + local.workers_group_defaults["launch_template_version"], + ) + } + + dynamic "override" { + for_each = lookup( + var.worker_groups_launch_template[count.index], + "override_instance_types", + local.workers_group_defaults["override_instance_types"] + ) + + content { + instance_type = override.value + } + } + + } + } + } + dynamic launch_template { + iterator = item + for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", null) != null) ? [] : list(var.worker_groups_launch_template[count.index]) + + content { + id = aws_launch_template.workers_launch_template.*.id[count.index] + version = lookup( + var.worker_groups_launch_template[count.index], + "launch_template_version", + local.workers_group_defaults["launch_template_version"], + ) + } } dynamic "initial_lifecycle_hook" { diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf deleted file mode 100644 index 3faa8a7ae8..0000000000 --- a/workers_launch_template_mixed.tf +++ /dev/null @@ -1,388 +0,0 @@ -# Worker Groups using Launch Templates with mixed instances policy - -resource "aws_autoscaling_group" "workers_launch_template_mixed" { - count = local.worker_group_launch_template_mixed_count - name_prefix = join( - "-", - compact( - [ - aws_eks_cluster.this.name, - lookup(var.worker_groups_launch_template_mixed[count.index], "name", count.index), - lookup(var.worker_groups_launch_template_mixed[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers_launch_template_mixed[count.index].id : "" - ] - ) - ) - - desired_capacity = lookup( - var.worker_groups_launch_template_mixed[count.index], - "asg_desired_capacity", - local.workers_group_defaults["asg_desired_capacity"], - ) - max_size = lookup( - var.worker_groups_launch_template_mixed[count.index], - "asg_max_size", - local.workers_group_defaults["asg_max_size"], - ) - min_size = lookup( - var.worker_groups_launch_template_mixed[count.index], - "asg_min_size", - local.workers_group_defaults["asg_min_size"], - ) - force_delete = lookup( - var.worker_groups_launch_template_mixed[count.index], - "asg_force_delete", - local.workers_group_defaults["asg_force_delete"], - ) - target_group_arns = lookup( - var.worker_groups_launch_template_mixed[count.index], - "target_group_arns", - local.workers_group_defaults["target_group_arns"] - ) - service_linked_role_arn = lookup( - var.worker_groups_launch_template_mixed[count.index], - "service_linked_role_arn", - local.workers_group_defaults["service_linked_role_arn"], - ) - vpc_zone_identifier = lookup( - var.worker_groups_launch_template_mixed[count.index], - "subnets", - local.workers_group_defaults["subnets"] - ) - protect_from_scale_in = lookup( - var.worker_groups_launch_template_mixed[count.index], - "protect_from_scale_in", - local.workers_group_defaults["protect_from_scale_in"], - ) - suspended_processes = lookup( - var.worker_groups_launch_template_mixed[count.index], - "suspended_processes", - local.workers_group_defaults["suspended_processes"] - ) - enabled_metrics = lookup( - var.worker_groups_launch_template_mixed[count.index], - "enabled_metrics", - local.workers_group_defaults["enabled_metrics"] - ) - placement_group = lookup( - var.worker_groups_launch_template_mixed[count.index], - "placement_group", - local.workers_group_defaults["placement_group"], - ) - termination_policies = lookup( - var.worker_groups_launch_template_mixed[count.index], - "termination_policies", - local.workers_group_defaults["termination_policies"] - ) - - mixed_instances_policy { - instances_distribution { - on_demand_allocation_strategy = lookup( - var.worker_groups_launch_template_mixed[count.index], - "on_demand_allocation_strategy", - local.workers_group_defaults["on_demand_allocation_strategy"], - ) - on_demand_base_capacity = lookup( - var.worker_groups_launch_template_mixed[count.index], - "on_demand_base_capacity", - local.workers_group_defaults["on_demand_base_capacity"], - ) - on_demand_percentage_above_base_capacity = lookup( - var.worker_groups_launch_template_mixed[count.index], - "on_demand_percentage_above_base_capacity", - local.workers_group_defaults["on_demand_percentage_above_base_capacity"], - ) - spot_allocation_strategy = lookup( - var.worker_groups_launch_template_mixed[count.index], - "spot_allocation_strategy", - local.workers_group_defaults["spot_allocation_strategy"], - ) - spot_instance_pools = lookup( - var.worker_groups_launch_template_mixed[count.index], - "spot_instance_pools", - local.workers_group_defaults["spot_instance_pools"], - ) - spot_max_price = lookup( - var.worker_groups_launch_template_mixed[count.index], - "spot_max_price", - local.workers_group_defaults["spot_max_price"], - ) - } - - launch_template { - launch_template_specification { - launch_template_id = aws_launch_template.workers_launch_template_mixed.*.id[count.index] - version = lookup( - var.worker_groups_launch_template_mixed[count.index], - "launch_template_version", - local.workers_group_defaults["launch_template_version"], - ) - } - - dynamic "override" { - for_each = lookup( - var.worker_groups_launch_template_mixed[count.index], - "override_instance_types", - local.workers_group_defaults["override_instance_types"] - ) - - content { - instance_type = override.value - } - } - - } - } - - dynamic "initial_lifecycle_hook" { - for_each = lookup(var.worker_groups_launch_template_mixed[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) - content { - name = lookup(initial_lifecycle_hook.value, "name", null) - default_result = lookup(initial_lifecycle_hook.value, "default_result", null) - heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) - lifecycle_transition = lookup(initial_lifecycle_hook.value, "lifecycle_transition", null) - notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) - notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) - role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) - } - } - - tags = concat( - [ - { - "key" = "Name" - "value" = "${aws_eks_cluster.this.name}-${lookup( - var.worker_groups_launch_template_mixed[count.index], - "name", - count.index, - )}-eks_asg" - "propagate_at_launch" = true - }, - { - "key" = "kubernetes.io/cluster/${aws_eks_cluster.this.name}" - "value" = "owned" - "propagate_at_launch" = true - }, - { - "key" = "k8s.io/cluster-autoscaler/${lookup( - var.worker_groups_launch_template_mixed[count.index], - "autoscaling_enabled", - local.workers_group_defaults["autoscaling_enabled"], - ) ? "enabled" : "disabled"}" - "value" = "true" - "propagate_at_launch" = false - }, - { - "key" = "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}" - "value" = aws_eks_cluster.this.name - "propagate_at_launch" = false - }, - { - "key" = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" - "value" = "${lookup( - var.worker_groups_launch_template_mixed[count.index], - "root_volume_size", - local.workers_group_defaults["root_volume_size"], - )}Gi" - "propagate_at_launch" = false - }, - ], - local.asg_tags, - lookup( - var.worker_groups_launch_template_mixed[count.index], - "tags", - local.workers_group_defaults["tags"] - ) - ) - - lifecycle { - create_before_destroy = true - ignore_changes = [desired_capacity] - } -} - -resource "aws_launch_template" "workers_launch_template_mixed" { - count = local.worker_group_launch_template_mixed_count - name_prefix = "${aws_eks_cluster.this.name}-${lookup( - var.worker_groups_launch_template_mixed[count.index], - "name", - count.index, - )}" - - network_interfaces { - associate_public_ip_address = lookup( - var.worker_groups_launch_template_mixed[count.index], - "public_ip", - local.workers_group_defaults["public_ip"], - ) - delete_on_termination = lookup( - var.worker_groups_launch_template_mixed[count.index], - "eni_delete", - local.workers_group_defaults["eni_delete"], - ) - security_groups = flatten([ - local.worker_security_group_id, - var.worker_additional_security_group_ids, - lookup( - var.worker_groups_launch_template_mixed[count.index], - "additional_security_group_ids", - local.workers_group_defaults["additional_security_group_ids"] - ) - ]) - } - - iam_instance_profile { - name = coalescelist( - aws_iam_instance_profile.workers_launch_template_mixed.*.name, - data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.name, - )[count.index] - } - - image_id = lookup( - var.worker_groups_launch_template_mixed[count.index], - "ami_id", - local.workers_group_defaults["ami_id"], - ) - instance_type = lookup( - var.worker_groups_launch_template_mixed[count.index], - "instance_type", - local.workers_group_defaults["instance_type"], - ) - key_name = lookup( - var.worker_groups_launch_template_mixed[count.index], - "key_name", - local.workers_group_defaults["key_name"], - ) - user_data = base64encode( - data.template_file.workers_launch_template_mixed.*.rendered[count.index], - ) - ebs_optimized = lookup( - var.worker_groups_launch_template_mixed[count.index], - "ebs_optimized", - lookup( - local.ebs_optimized, - lookup( - var.worker_groups_launch_template_mixed[count.index], - "instance_type", - local.workers_group_defaults["instance_type"], - ), - false, - ), - ) - - credit_specification { - cpu_credits = lookup( - var.worker_groups_launch_template_mixed[count.index], - "cpu_credits", - local.workers_group_defaults["cpu_credits"] - ) - } - - monitoring { - enabled = lookup( - var.worker_groups_launch_template_mixed[count.index], - "enable_monitoring", - local.workers_group_defaults["enable_monitoring"], - ) - } - - placement { - tenancy = lookup( - var.worker_groups_launch_template_mixed[count.index], - "launch_template_placement_tenancy", - local.workers_group_defaults["launch_template_placement_tenancy"], - ) - group_name = lookup( - var.worker_groups_launch_template_mixed[count.index], - "launch_template_placement_group", - local.workers_group_defaults["launch_template_placement_group"], - ) - } - - block_device_mappings { - device_name = lookup( - var.worker_groups_launch_template_mixed[count.index], - "root_block_device_name", - local.workers_group_defaults["root_block_device_name"], - ) - - ebs { - volume_size = lookup( - var.worker_groups_launch_template_mixed[count.index], - "root_volume_size", - local.workers_group_defaults["root_volume_size"], - ) - volume_type = lookup( - var.worker_groups_launch_template_mixed[count.index], - "root_volume_type", - local.workers_group_defaults["root_volume_type"], - ) - iops = lookup( - var.worker_groups_launch_template_mixed[count.index], - "root_iops", - local.workers_group_defaults["root_iops"], - ) - encrypted = lookup( - var.worker_groups_launch_template_mixed[count.index], - "root_encrypted", - local.workers_group_defaults["root_encrypted"], - ) - kms_key_id = lookup( - var.worker_groups_launch_template_mixed[count.index], - "root_kms_key_id", - local.workers_group_defaults["root_kms_key_id"], - ) - delete_on_termination = true - } - } - - tag_specifications { - resource_type = "volume" - - tags = merge( - { - "Name" = "${aws_eks_cluster.this.name}-${lookup( - var.worker_groups_launch_template_mixed[count.index], - "name", - count.index, - )}-eks_asg" - }, - var.tags, - ) - } - - tags = var.tags - - lifecycle { - create_before_destroy = true - } -} - -resource "random_pet" "workers_launch_template_mixed" { - count = local.worker_group_launch_template_mixed_count - - separator = "-" - length = 2 - - keepers = { - lt_name = join( - "-", - compact( - [ - aws_launch_template.workers_launch_template_mixed[count.index].name, - aws_launch_template.workers_launch_template_mixed[count.index].latest_version - ] - ) - ) - } -} - -resource "aws_iam_instance_profile" "workers_launch_template_mixed" { - count = var.manage_worker_iam_resources ? local.worker_group_launch_template_mixed_count : 0 - name_prefix = aws_eks_cluster.this.name - role = lookup( - var.worker_groups_launch_template_mixed[count.index], - "iam_role_id", - local.default_iam_role_id, - ) - path = var.iam_path -} From f88220a8eb45d30a2f5ba7ed543466441a7e5eef Mon Sep 17 00:00:00 2001 From: Marcello Romani Date: Fri, 13 Sep 2019 15:52:10 +0100 Subject: [PATCH 056/962] Fix link to docs so it doesn't 404 when published on terraform registry page (#507) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f567503c6b..36374510e0 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,9 @@ module "my-cluster" { ## Other documentation -- [Autoscaling](docs/autoscaling.md): How to enable worker node autoscaling. -- [Enable Docker Bridge Network](docs/enable-docker-bridge-network.md): How to enable the docker bridge network when using the EKS-optimized AMI, which disables it by default. -- [Spot instances](docs/spot-instances.md): How to use spot instances with this module. +* [Autoscaling](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/autoscaling.md): How to enable worker node autoscaling. +* [Enable Docker Bridge Network](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/enable-docker-bridge-network.md): How to enable the docker bridge network when using the EKS-optimized AMI, which disables it by default. +* [Spot instances](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/spot-instances.md): How to use spot instances with this module. ## Release schedule From 4f4d9c304d4978bfe569f5183b9829a7fb30b582 Mon Sep 17 00:00:00 2001 From: Tomas Ostasevicius Date: Mon, 16 Sep 2019 11:32:39 +0200 Subject: [PATCH 057/962] Fix launch template market option expansion (#508) * Fix launch template market option expansion * Update changelog --- CHANGELOG.md | 3 ++- workers_launch_template.tf | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f503c45e3..6df1af4de9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,8 +30,9 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fixed errors sometimes happening during destroy due to usage of coalesce() in local.tf (by @petrikero) - Removed historical mention of adding caller's IPv4 to cluster security group (by @dpiddockcmp) - Wrapped `kubelet_extra_args` in double quotes instead of singe quotes (by @nxf5025) - - Write your awesome change here (by @you) - Make terraform plan more consistent and avoid unnecessary "(known after apply)" (by @barryib) + - Made sure that `market_type` was correctly passed to `workers_launch_template` (by @to266) + - Write your awesome change here (by @you) # History diff --git a/workers_launch_template.tf b/workers_launch_template.tf index ce1c21d4a7..258d536f09 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -316,10 +316,9 @@ resource "aws_launch_template" "workers_launch_template" { } dynamic instance_market_options { - iterator = item for_each = lookup(var.worker_groups_launch_template[count.index], "market_type", null) == null ? [] : list(lookup(var.worker_groups_launch_template[count.index], "market_type", null)) content { - market_type = item.value + market_type = instance_market_options.value } } From 2b88e506e304f5dddf3707f9ef44fcf5ac6c6a82 Mon Sep 17 00:00:00 2001 From: russwhelan <30686539+russwhelan@users.noreply.github.com> Date: Mon, 16 Sep 2019 10:45:35 +0100 Subject: [PATCH 058/962] Add URL for OIDC issuer to allow IAM roles for Pods (#506) * Add URL for OIDC issuer to allow IAM roles for Pods * Update Documentation --- CHANGELOG.md | 1 + README.md | 1 + outputs.tf | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6df1af4de9..99d795a610 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added support for initial lifecycle hooks for autosacling groups (@barryib) - Added option to recreate ASG when LT or LC changes (by @barryib) - Ability to specify workers role name (by @ivanich) + - Added output for OIDC Issuer URL (by @russwhelan) - Added support for Mixed Instance ASG using `worker_groups_launch_template` variable (by @sppwf) - Changed ASG Tags generation using terraform 12 `for` utility (by @sppwf) - Removed `worker_groups_launch_template_mixed` variable (by @sppwf) diff --git a/README.md b/README.md index 36374510e0..27aa66aeeb 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_id | The name/id of the EKS cluster. | | cluster\_security\_group\_id | Security group ID attached to the EKS cluster. | | cluster\_version | The Kubernetes server version for the EKS cluster. | +| cluster\_oidc\_issuer\_url | The URL on the EKS cluster OIDC Issuer. | | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | diff --git a/outputs.tf b/outputs.tf index 8f48268f07..24a3949b95 100644 --- a/outputs.tf +++ b/outputs.tf @@ -43,6 +43,11 @@ output "cluster_iam_role_arn" { value = local.cluster_iam_role_arn } +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster OIDC Issuer" + value = aws_eks_cluster.this.identity.0.oidc.0.issuer +} + output "cloudwatch_log_group_name" { description = "Name of cloudwatch log group created" value = aws_cloudwatch_log_group.this.*.name From 13ee0914b91ff5d528738f2ae7295512aeb4c41b Mon Sep 17 00:00:00 2001 From: Ali Kahoot Date: Tue, 17 Sep 2019 18:10:13 +0500 Subject: [PATCH 059/962] add doc for iam permission (#511) * add doc for minimum iam permission Signed-off-by: kahootali * refer iam doc in readme Signed-off-by: kahootali * remove iam permissions from examples * set alphabetically the iam permissons --- README.md | 1 + docs/iam-permissions.md | 121 ++++++++++++++++++++++++++++++++++++++++ examples/README.md | 112 +------------------------------------ 3 files changed, 123 insertions(+), 111 deletions(-) create mode 100644 docs/iam-permissions.md diff --git a/README.md b/README.md index 27aa66aeeb..1d219b6171 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ module "my-cluster" { * [Autoscaling](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/autoscaling.md): How to enable worker node autoscaling. * [Enable Docker Bridge Network](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/enable-docker-bridge-network.md): How to enable the docker bridge network when using the EKS-optimized AMI, which disables it by default. * [Spot instances](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/spot-instances.md): How to use spot instances with this module. +* [IAM Permissions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md): Minimum IAM permissions needed to setup EKS Cluster. ## Release schedule diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md new file mode 100644 index 0000000000..4fbc828447 --- /dev/null +++ b/docs/iam-permissions.md @@ -0,0 +1,121 @@ +# IAM Permissions + +Following IAM permissions are the minimum permissions needed for your IAM user or IAM role to create an EKS cluster. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "autoscaling:AttachInstances", + "autoscaling:CreateAutoScalingGroup", + "autoscaling:CreateLaunchConfiguration", + "autoscaling:CreateOrUpdateTags", + "autoscaling:DeleteAutoScalingGroup", + "autoscaling:DeleteLaunchConfiguration", + "autoscaling:DeleteTags", + "autoscaling:Describe*", + "autoscaling:DetachInstances", + "autoscaling:SetDesiredCapacity", + "autoscaling:UpdateAutoScalingGroup", + "autoscaling:SuspendProcesses", + "ec2:AllocateAddress", + "ec2:AssignPrivateIpAddresses", + "ec2:Associate*", + "ec2:AttachInternetGateway", + "ec2:AttachNetworkInterface", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateDefaultSubnet", + "ec2:CreateDhcpOptions", + "ec2:CreateEgressOnlyInternetGateway", + "ec2:CreateInternetGateway", + "ec2:CreateNatGateway", + "ec2:CreateNetworkInterface", + "ec2:CreateRoute", + "ec2:CreateRouteTable", + "ec2:CreateSecurityGroup", + "ec2:CreateSubnet", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:CreateVpc", + "ec2:DeleteDhcpOptions", + "ec2:DeleteEgressOnlyInternetGateway", + "ec2:DeleteInternetGateway", + "ec2:DeleteNatGateway", + "ec2:DeleteNetworkInterface", + "ec2:DeleteRoute", + "ec2:DeleteRouteTable", + "ec2:DeleteSecurityGroup", + "ec2:DeleteSubnet", + "ec2:DeleteTags", + "ec2:DeleteVolume", + "ec2:DeleteVpc", + "ec2:DeleteVpnGateway", + "ec2:Describe*", + "ec2:DetachInternetGateway", + "ec2:DetachNetworkInterface", + "ec2:DetachVolume", + "ec2:Disassociate*", + "ec2:ModifySubnetAttribute", + "ec2:ModifyVpcAttribute", + "ec2:ModifyVpcEndpoint", + "ec2:ReleaseAddress", + "ec2:RevokeSecurityGroupEgress", + "ec2:RevokeSecurityGroupIngress", + "ec2:UpdateSecurityGroupRuleDescriptionsEgress", + "ec2:UpdateSecurityGroupRuleDescriptionsIngress", + "ec2:CreateLaunchTemplate", + "ec2:CreateLaunchTemplateVersion", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:GetLaunchTemplateData", + "ec2:ModifyLaunchTemplate", + "ec2:RunInstances", + "eks:CreateCluster", + "eks:DeleteCluster", + "eks:DescribeCluster", + "eks:ListClusters", + "eks:UpdateClusterConfig", + "eks:DescribeUpdate", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreateServiceLinkedRole", + "iam:CreatePolicy", + "iam:CreatePolicyVersion", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeleteRole", + "iam:DeleteRolePolicy", + "iam:DeleteServiceLinkedRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:List*", + "iam:PassRole", + "iam:PutRolePolicy", + "iam:RemoveRoleFromInstanceProfile", + "iam:TagRole", + "iam:UpdateAssumeRolePolicy", + // Following permissions are needed if cluster_enabled_log_types is enabled + "logs:CreateLogGroup", + "logs:DescribeLogGroups", + "logs:DeleteLogGroup", + "logs:ListTagsLogGroup", + "logs:PutRetentionPolicy" + ], + "Resource": "*" + } + ] +} +``` \ No newline at end of file diff --git a/examples/README.md b/examples/README.md index 1217618d98..7ee5bebfb9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,114 +8,4 @@ These serve a few purposes: ## IAM Permissions -The following IAM policy is the minimum needed to execute the module from the test suite. - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "autoscaling:AttachInstances", - "autoscaling:CreateAutoScalingGroup", - "autoscaling:CreateLaunchConfiguration", - "autoscaling:CreateOrUpdateTags", - "autoscaling:DeleteAutoScalingGroup", - "autoscaling:DeleteLaunchConfiguration", - "autoscaling:DeleteTags", - "autoscaling:Describe*", - "autoscaling:DetachInstances", - "autoscaling:SetDesiredCapacity", - "autoscaling:UpdateAutoScalingGroup", - "autoscaling:SuspendProcesses", - "ec2:AllocateAddress", - "ec2:AssignPrivateIpAddresses", - "ec2:Associate*", - "ec2:AttachInternetGateway", - "ec2:AttachNetworkInterface", - "ec2:AuthorizeSecurityGroupEgress", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateDefaultSubnet", - "ec2:CreateDhcpOptions", - "ec2:CreateEgressOnlyInternetGateway", - "ec2:CreateInternetGateway", - "ec2:CreateNatGateway", - "ec2:CreateNetworkInterface", - "ec2:CreateRoute", - "ec2:CreateRouteTable", - "ec2:CreateSecurityGroup", - "ec2:CreateSubnet", - "ec2:CreateTags", - "ec2:CreateVolume", - "ec2:CreateVpc", - "ec2:DeleteDhcpOptions", - "ec2:DeleteEgressOnlyInternetGateway", - "ec2:DeleteInternetGateway", - "ec2:DeleteNatGateway", - "ec2:DeleteNetworkInterface", - "ec2:DeleteRoute", - "ec2:DeleteRouteTable", - "ec2:DeleteSecurityGroup", - "ec2:DeleteSubnet", - "ec2:DeleteTags", - "ec2:DeleteVolume", - "ec2:DeleteVpc", - "ec2:DeleteVpnGateway", - "ec2:Describe*", - "ec2:DetachInternetGateway", - "ec2:DetachNetworkInterface", - "ec2:DetachVolume", - "ec2:Disassociate*", - "ec2:ModifySubnetAttribute", - "ec2:ModifyVpcAttribute", - "ec2:ModifyVpcEndpoint", - "ec2:ReleaseAddress", - "ec2:RevokeSecurityGroupEgress", - "ec2:RevokeSecurityGroupIngress", - "ec2:UpdateSecurityGroupRuleDescriptionsEgress", - "ec2:UpdateSecurityGroupRuleDescriptionsIngress", - "ec2:CreateLaunchTemplate", - "ec2:CreateLaunchTemplateVersion", - "ec2:DeleteLaunchTemplate", - "ec2:DeleteLaunchTemplateVersions", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeLaunchTemplateVersions", - "ec2:GetLaunchTemplateData", - "ec2:ModifyLaunchTemplate", - "ec2:RunInstances", - "eks:CreateCluster", - "eks:DeleteCluster", - "eks:DescribeCluster", - "eks:ListClusters", - "eks:UpdateClusterConfig", - "eks:DescribeUpdate", - "iam:AddRoleToInstanceProfile", - "iam:AttachRolePolicy", - "iam:CreateInstanceProfile", - "iam:CreatePolicy", - "iam:CreatePolicyVersion", - "iam:CreateRole", - "iam:DeleteInstanceProfile", - "iam:DeletePolicy", - "iam:DeleteRole", - "iam:DeleteRolePolicy", - "iam:DeleteServiceLinkedRole", - "iam:DetachRolePolicy", - "iam:GetInstanceProfile", - "iam:GetPolicy", - "iam:GetPolicyVersion", - "iam:GetRole", - "iam:GetRolePolicy", - "iam:List*", - "iam:PassRole", - "iam:PutRolePolicy", - "iam:RemoveRoleFromInstanceProfile", - "iam:UpdateAssumeRolePolicy" - ], - "Resource": "*" - } - ] -} -``` +You can see the minimum IAM Permissions required [here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md). From bf8c324761e2efaff95c85a6e9e3b017a49f57dc Mon Sep 17 00:00:00 2001 From: Max Williams Date: Wed, 18 Sep 2019 10:01:38 +0200 Subject: [PATCH 060/962] New Release v6.0.0 (#514) * New Release v6.0.0 * bump required AWS provider version and regeneration of variables/outputs --- CHANGELOG.md | 19 +++++++++++++------ README.md | 29 ++++++++++++----------------- examples/basic/main.tf | 2 +- examples/launch_templates/main.tf | 2 +- examples/spot_instances/main.tf | 2 +- version | 2 +- versions.tf | 2 +- 7 files changed, 30 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99d795a610..07605e4724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,20 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v6.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...HEAD)] - 2019-08-??] +## [[v6.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...HEAD)] - 2019-10-??] ### Added +- Write your awesome addition here (by @you) + +### Changed + +- Write your awesome change here (by @you) + +# History + +## [[v6.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v6.0.0)] - 2019-09-17] + - Added `market_type` to `workers_launch_template.tf` allow the usage of spot nodegroups without mixed instances policy. - Added support for log group tag in `./cluster.tf` (@lucas-giaco) - Added support for workers iam role tag in `./workers.tf` (@lucas-giaco) @@ -22,20 +32,17 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added output for OIDC Issuer URL (by @russwhelan) - Added support for Mixed Instance ASG using `worker_groups_launch_template` variable (by @sppwf) - Changed ASG Tags generation using terraform 12 `for` utility (by @sppwf) - - Removed `worker_groups_launch_template_mixed` variable (by @sppwf) + - **Breaking:** Removed `worker_groups_launch_template_mixed` variable (by @sppwf) ### Changed - Update to EKS 1.14 (by @nauxliu) - - Support map users and roles to multiple groups (by @nauxliu) + - **Breaking:** Support map users and roles to multiple groups (by @nauxliu) - Fixed errors sometimes happening during destroy due to usage of coalesce() in local.tf (by @petrikero) - Removed historical mention of adding caller's IPv4 to cluster security group (by @dpiddockcmp) - Wrapped `kubelet_extra_args` in double quotes instead of singe quotes (by @nxf5025) - Make terraform plan more consistent and avoid unnecessary "(known after apply)" (by @barryib) - Made sure that `market_type` was correctly passed to `workers_launch_template` (by @to266) - - Write your awesome change here (by @you) - -# History ## [[v5.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v5.1.1)] - 2019-07-30] diff --git a/README.md b/README.md index 1d219b6171..cb4448387b 100644 --- a/README.md +++ b/README.md @@ -53,12 +53,6 @@ module "my-cluster" { * [Spot instances](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/spot-instances.md): How to use spot instances with this module. * [IAM Permissions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md): Minimum IAM permissions needed to setup EKS Cluster. -## Release schedule - -Generally the maintainers will try to release the module once every 2 weeks to -keep up with PR additions. If particularly pressing changes are added or maintainers -come up with the spare time (hah!), release may happen more often on occasion. - ## Testing This module has been packaged with [awspec](https://github.com/k1LoW/awspec) tests through [kitchen](https://kitchen.ci/) and [kitchen-terraform](https://newcontext-oss.github.io/kitchen-terraform/). To run them: @@ -101,7 +95,8 @@ The [changelog](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/ ## Authors -Created and maintained by [Brandon O'Connor](https://github.com/brandoconnor) - brandon@atscale.run. +Created by [Brandon O'Connor](https://github.com/brandoconnor) - brandon@atscale.run. +Maintained by [Max Williams](https://github.com/max-rocket-internet) Many thanks to [the contributors listed here](https://github.com/terraform-aws-modules/terraform-aws-eks/graphs/contributors)! ## License @@ -119,7 +114,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | list(string) | `[]` | no | | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | bool | `"false"` | no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | bool | `"true"` | no | -| cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. | string | `""` | no | +| cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. | string | `""` | no | | cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | string | `""` | no | | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | @@ -127,7 +122,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.14"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Should end in a forward slash `/` . | string | `"./"` | no | | iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | -| kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. \["-r", "MyEksRole"\]. | list(string) | `[]` | no | +| kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | | kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | string | `"aws-iam-authenticator"` | no | | kubeconfig\_aws\_authenticator\_command\_args | Default arguments passed to the authenticator command. Defaults to [token -i $cluster_name]. | list(string) | `[]` | no | | kubeconfig\_aws\_authenticator\_env\_variables | Environment variables that should be used when executing the authenticator. e.g. { AWS_PROFILE = "eks"}. | map(string) | `{}` | no | @@ -136,23 +131,23 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | manage\_aws\_auth | Whether to apply the aws-auth configmap file. | string | `"true"` | no | | manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | bool | `"true"` | no | | manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | bool | `"true"` | no | -| workers\_role\_name | User defined workers role name. | sting | `""` | no | | map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | -| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(map(string)) | `[]` | no | -| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(map(string)) | `[]` | no | +| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | +| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `""` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | | tags | A map of tags to add to all resources. | map(string) | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | -| worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster\_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | +| worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | -| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers\_group\_defaults for valid keys. | any | `[]` | no | -| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | any | `[]` | no | +| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | +| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | | worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster. | string | `""` | no | | worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | number | `"1025"` | no | | workers\_additional\_policies | Additional policies to be added to workers | list(string) | `[]` | no | -| workers\_group\_defaults | Override default values for target groups. See workers\_group\_defaults\_defaults in local.tf for valid keys. | any | `{}` | no | +| workers\_group\_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | any | `{}` | no | +| workers\_role\_name | User defined workers role name. | string | `""` | no | | write\_aws\_auth\_config | Whether to write the aws-auth configmap file. | bool | `"true"` | no | | write\_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | bool | `"true"` | no | @@ -167,9 +162,9 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_iam\_role\_arn | IAM role ARN of the EKS cluster. | | cluster\_iam\_role\_name | IAM role name of the EKS cluster. | | cluster\_id | The name/id of the EKS cluster. | +| cluster\_oidc\_issuer\_url | The URL on the EKS cluster OIDC Issuer | | cluster\_security\_group\_id | Security group ID attached to the EKS cluster. | | cluster\_version | The Kubernetes server version for the EKS cluster. | -| cluster\_oidc\_issuer\_url | The URL on the EKS cluster OIDC Issuer. | | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | diff --git a/examples/basic/main.tf b/examples/basic/main.tf index e68fd79c87..7e282333d9 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -3,7 +3,7 @@ terraform { } provider "aws" { - version = ">= 2.11" + version = ">= 2.28.1" region = var.region } diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 51148aac01..008a4f32df 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -3,7 +3,7 @@ terraform { } provider "aws" { - version = ">= 2.11" + version = ">= 2.28.1" region = var.region } diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 0a73642cee..741531d41c 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -3,7 +3,7 @@ terraform { } provider "aws" { - version = ">= 2.11" + version = ">= 2.28.1" region = var.region } diff --git a/version b/version index 6b7ace3e91..9773998bc1 100644 --- a/version +++ b/version @@ -1 +1 @@ -v5.1.0 +v6.0.0 diff --git a/versions.tf b/versions.tf index aff0bf8375..5e2d714efc 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.12" required_providers { - aws = ">= 2.8" + aws = ">= 2.28.1" local = ">= 1.2" null = ">= 2.1" template = ">= 2.1" From b6125b8501e8e0b4f0c070b1e52daa84395aa032 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 18 Sep 2019 16:43:01 +0200 Subject: [PATCH 061/962] output empty string when cluster identity is empty (#516) --- outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/outputs.tf b/outputs.tf index 24a3949b95..3a69d502b3 100644 --- a/outputs.tf +++ b/outputs.tf @@ -45,7 +45,7 @@ output "cluster_iam_role_arn" { output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster OIDC Issuer" - value = aws_eks_cluster.this.identity.0.oidc.0.issuer + value = concat(aws_eks_cluster.this.identity.*.oidc.0.issuer, [""])[0] } output "cloudwatch_log_group_name" { From 0d49f8265af2bdbd1ace6c23cfab79d37ef7bc94 Mon Sep 17 00:00:00 2001 From: irti Date: Thu, 19 Sep 2019 00:13:20 +0500 Subject: [PATCH 062/962] [add-notes] update local.tf file --- local.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/local.tf b/local.tf index e5a9c33b6c..f81817567a 100644 --- a/local.tf +++ b/local.tf @@ -23,9 +23,9 @@ locals { name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. tags = [] # A list of map defining extra tags to be applied to the worker group ASG. ami_id = data.aws_ami.eks_worker.id # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI. - asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group. + asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. asg_max_size = "3" # Maximum worker capacity in the autoscaling group. - asg_min_size = "1" # Minimum worker capacity in the autoscaling group. + asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desirec_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. asg_force_delete = false # Enable forced deletion for the autoscaling group. asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. asg_recreate_on_change = false # Recreate the autoscaling group when LT or LC change. From 347db3e23053a3c11fd74eb32c7a65e09730962b Mon Sep 17 00:00:00 2001 From: Xiangxuan Liu Date: Wed, 11 Sep 2019 23:17:26 +0800 Subject: [PATCH 063/962] Improve CI pipeline --- .github/workflows/lint.yml | 76 ++++++++++++++++++++++---------------- README.md | 4 +- variables.tf | 2 +- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0cafaedd9f..53cac00ad1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,40 +2,52 @@ name: Lint on: [push, pull_request] jobs: + tflint: + name: TFLint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: TFLint + uses: docker://wata727/tflint + + fmt: + name: Code Format + runs-on: ubuntu-latest + container: + image: hashicorp/terraform:latest + steps: + - uses: actions/checkout@master + - run: terraform fmt --recursive -check=true + + docs: + name: Docs + runs-on: macOS-latest + steps: + - uses: actions/checkout@master + - name: Install Deps + run: brew install pre-commit terraform-docs terraform + - name: Check Docs + run: pre-commit run --show-diff-on-failure --all-files terraform_docs + validate: name: Validate runs-on: ubuntu-latest + container: + image: hashicorp/terraform:latest steps: - - uses: actions/checkout@master - # - uses: actions/setup-ruby@v1 - # with: - # ruby-version: '2.4.6' - # - name: Install Kitchen-Terraform - # run: | - # gem install bundler --no-document - # bundle install - - name: Install Terraform - run: | - export TERRAFORM_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version') - curl --silent --output terraform.zip "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - unzip terraform.zip ; rm -f terraform.zip; chmod +x terraform - mkdir -p ${HOME}/bin ; export PATH=${PATH}:${HOME}/bin; mv terraform ${HOME}/bin/ - terraform -v - - name: Validate Code - env: - AWS_REGION: 'us-east-1' - TF_VAR_region: ${AWS_REGION} - TF_WARN_OUTPUT_ERRORS: 1 - run: | - terraform init - terraform fmt -check=true - terraform validate -var "region=${AWS_REGION}" -var "vpc_id=vpc-123456" -var "subnets=[\"subnet-12345a\"]" -var "workers_ami_id=ami-123456" -var "cluster_ingress_cidrs=[]" -var "cluster_name=test_cluster" - - name: Validate Examples - run: | - for example in $(find examples -maxdepth 1 -mindepth 1 -type d); do - cd $example + - uses: actions/checkout@master + - name: Validate Code + env: + AWS_REGION: 'us-east-1' + TF_WARN_OUTPUT_ERRORS: 1 + run: | terraform init - terraform fmt -check=true - terraform validate - cd - - done + terraform validate -var "region=${AWS_REGION}" -var "vpc_id=vpc-123456" -var "subnets=[\"subnet-12345a\"]" -var "workers_ami_id=ami-123456" -var "cluster_ingress_cidrs=[]" -var "cluster_name=test_cluster" + - name: Validate Examples + run: | + for example in $(find examples -maxdepth 1 -mindepth 1 -type d); do + cd $example + terraform init + terraform validate + cd - + done diff --git a/README.md b/README.md index cb4448387b..c35e307b16 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | | map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | | map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | -| permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `""` | no | +| permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `"null"` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | | tags | A map of tags to add to all resources. | map(string) | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | @@ -181,4 +181,4 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | workers\_launch\_template\_latest\_versions | Latest versions of the worker launch templates. | | workers\_user\_data | User data of worker groups | - + \ No newline at end of file diff --git a/variables.tf b/variables.tf index bb8a7ffd0b..d1a1f7173e 100644 --- a/variables.tf +++ b/variables.tf @@ -207,7 +207,7 @@ variable "worker_create_security_group" { variable "permissions_boundary" { description = "If provided, all IAM roles will be created with this permissions boundary attached." type = string - default = "" + default = null } variable "iam_path" { From 0fd70461203b96017569e3ee159cf5f98a21b993 Mon Sep 17 00:00:00 2001 From: Irtiza Ali Date: Thu, 19 Sep 2019 17:33:09 +0500 Subject: [PATCH 064/962] update local.tf by fixing typo --- local.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local.tf b/local.tf index f81817567a..a8c3e6219e 100644 --- a/local.tf +++ b/local.tf @@ -25,7 +25,7 @@ locals { ami_id = data.aws_ami.eks_worker.id # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI. asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. asg_max_size = "3" # Maximum worker capacity in the autoscaling group. - asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desirec_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. + asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. asg_force_delete = false # Enable forced deletion for the autoscaling group. asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. asg_recreate_on_change = false # Recreate the autoscaling group when LT or LC change. From 185cd4499eb802e8e807c3c6bc3e065c4edceffa Mon Sep 17 00:00:00 2001 From: irti Date: Sat, 21 Sep 2019 00:12:21 +0500 Subject: [PATCH 065/962] [add-notes] update the docs/autoscaling.md document --- docs/autoscaling.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/autoscaling.md b/docs/autoscaling.md index 35ecc3154d..c4b6d98184 100644 --- a/docs/autoscaling.md +++ b/docs/autoscaling.md @@ -28,6 +28,10 @@ To install the chart, simply run helm with the `--values` option: helm install stable/cluster-autoscaler --values=path/to/your/values-file.yaml ``` +`NOTE` + +There is a variable `asg_desired_capacity` given in the `local.tf` file, currently it can be used to change the desired worker(s) capacity in the autoscaling group but currently it is being ignored in terraform to reduce the [complexities](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/510#issuecomment-531700442) and the feature of scaling up and down the cluster nodes is being handled by the cluster autoscaler. + ## See More [Using AutoScalingGroup MixedInstancesPolicy](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-autoscalinggroup-mixedinstancespolicy) \ No newline at end of file From f79c79015320b3089a82ce0d83fb625472c801c1 Mon Sep 17 00:00:00 2001 From: Roman Voitenko Date: Mon, 23 Sep 2019 17:50:01 +0200 Subject: [PATCH 066/962] Gpu workers support (#515) * added posibility to use Amazon EKS-Optimized AMI with GPU Support for workers * updated CHANGELOG.md * added missing var prefix * updated Changelog to ver 6.x.x --- CHANGELOG.md | 2 +- README.md | 3 ++- data.tf | 2 +- variables.tf | 6 ++++++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07605e4724..8c298d00b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added -- Write your awesome addition here (by @you) + - Added support for different workers AMI's, i.e. with GPU support (by @rvoitenko) ### Changed diff --git a/README.md b/README.md index c35e307b16..3a607f8cc3 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | tags | A map of tags to add to all resources. | map(string) | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | -| worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | +| worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster\_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | +| worker\_ami\_name\_filter\_prefix | Name prefix filter for AWS EKS worker AMI. Default behaviour will get regular EKS-Optimized AMI but could be set to a EKS-Optimized AMI with GPU Support, e.g. "amazon-eks-gpu-node", or custom AMI | string | `"amazon-eks-node"` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | diff --git a/data.tf b/data.tf index c1f92ca4e1..3a5ae4fcb4 100644 --- a/data.tf +++ b/data.tf @@ -19,7 +19,7 @@ data "aws_iam_policy_document" "workers_assume_role_policy" { data "aws_ami" "eks_worker" { filter { name = "name" - values = ["amazon-eks-node-${var.cluster_version}-${var.worker_ami_name_filter}"] + values = ["${var.worker_ami_name_filter_prefix}-${var.cluster_version}-${var.worker_ami_name_filter}"] } most_recent = true diff --git a/variables.tf b/variables.tf index d1a1f7173e..47d49e79c5 100644 --- a/variables.tf +++ b/variables.tf @@ -126,6 +126,12 @@ variable "worker_ami_name_filter" { default = "v*" } +variable "worker_ami_name_filter_prefix" { + description = "Name prefix filter for AWS EKS worker AMI. Default behaviour will get regular EKS-Optimized AMI but could be set to a EKS-Optimized AMI with GPU Support, e.g. \"amazon-eks-gpu-node\", or custom AMI" + type = string + default = "amazon-eks-node" +} + variable "worker_additional_security_group_ids" { description = "A list of additional security group ids to attach to worker instances" type = list(string) From 88afa4338dc342be75a5729a751a1c1c761267f2 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 24 Sep 2019 10:02:02 +0200 Subject: [PATCH 067/962] fix docs generation for lint (#523) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a607f8cc3..18c59234c7 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | tags | A map of tags to add to all resources. | map(string) | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | -| worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster\_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | +| worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | | worker\_ami\_name\_filter\_prefix | Name prefix filter for AWS EKS worker AMI. Default behaviour will get regular EKS-Optimized AMI but could be set to a EKS-Optimized AMI with GPU Support, e.g. "amazon-eks-gpu-node", or custom AMI | string | `"amazon-eks-node"` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | From 39cb93b1dad13ec86e311a9d130858e82b423d0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatu=20Sepp=C3=A4-Lassila?= Date: Tue, 24 Sep 2019 12:09:44 +0300 Subject: [PATCH 068/962] Use null as default value for target group ARNs (#524) * Use null as default value for target group ARNs * Update CHANGELOG close #499 --- CHANGELOG.md | 2 +- local.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c298d00b9..7f442e5066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed -- Write your awesome change here (by @you) +- Use null as default value for `target_group_arns` attribute of worker autoscaling group (by @tatusl) # History diff --git a/local.tf b/local.tf index a8c3e6219e..c388221961 100644 --- a/local.tf +++ b/local.tf @@ -50,7 +50,7 @@ locals { iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. iam_role_id = "local.default_iam_role_id" # A custom IAM role id. Incompatible with iam_instance_profile_name. Literal local.default_iam_role_id will never be used but if iam_role_id is not set, the local.default_iam_role_id interpolation will be used. suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] - target_group_arns = [] # A list of ALB target group ARNs to be associated to the ASG + target_group_arns = null # A list of ALB target group ARNs to be associated to the ASG enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] placement_group = "" # The name of the placement group into which to launch the instances, if any. service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS From 07c61b6e26b812310bda08953f5cf4e4ad13f385 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 25 Sep 2019 11:19:16 +0200 Subject: [PATCH 069/962] New release 6.0.1 (#528) * New release 6.0.1 --- CHANGELOG.md | 13 +++++++++++++ version | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f442e5066..8222332fc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,18 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v6.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...HEAD)] - 2019-10-??] +### Added + +- Write your awesome addition here (by @you) + +### Changed + + - Write your awesome change here (by @you) + +# History + +## [[v6.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...v6.0.1)] - 2019-09-25] + ### Added - Added support for different workers AMI's, i.e. with GPU support (by @rvoitenko) @@ -16,6 +28,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed - Use null as default value for `target_group_arns` attribute of worker autoscaling group (by @tatusl) +- Output empty string when cluster identity is empty (by @tbarry) # History diff --git a/version b/version index 9773998bc1..23e595e7bd 100644 --- a/version +++ b/version @@ -1 +1 @@ -v6.0.0 +v6.0.1 From 27e6e3d7a993cb6b12ae60aed49dc6f04f4718dd Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Thu, 26 Sep 2019 13:43:00 +0200 Subject: [PATCH 070/962] First FAQ draft (#526) * First FAQ draft * Add aws-auth configmap failure * Mention ASG auto recreation --- .github/ISSUE_TEMPLATE.md | 2 +- README.md | 1 + docs/faq.md | 98 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 docs/faq.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 8243a0028a..bcb7188d94 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,7 +4,7 @@ * [ ] bug report * [ ] feature request -* [ ] support request +* [ ] support request - read the [FAQ](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md) first! * [ ] kudos, thank you, warm fuzzy ## What is the current behavior? diff --git a/README.md b/README.md index 18c59234c7..1c338feca1 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ module "my-cluster" { * [Enable Docker Bridge Network](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/enable-docker-bridge-network.md): How to enable the docker bridge network when using the EKS-optimized AMI, which disables it by default. * [Spot instances](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/spot-instances.md): How to use spot instances with this module. * [IAM Permissions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md): Minimum IAM permissions needed to setup EKS Cluster. +* [FAQ](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md): Frequently Asked Questions ## Testing diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000000..8aa2511b0d --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,98 @@ +# Frequently Asked Questions + +## How do I customize X on the worker group's settings? + +All the options that can be customized for worker groups are listed in [local.tf](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/local.tf) under `workers_group_defaults_defaults`. + +Please open Issues or PRs if you think something is missing. + +## Why are nodes not being registered? + +### Networking + +Often caused by a networking or endpoint configuration issue. + +At least one of the cluster public or private endpoints must be enabled in order for access to the cluster to work. + +Your nodes need to be able to contact the EKS cluster endpoint. By default the module only creates a public endpoint. You should also enable the private endpoint by setting `cluster_endpoint_private_access = true` on this module. + +If you have the private endpoint enabled ensure that you also have VPC DNS enabled. Set `enable_dns_hostnames = true` on your `aws_vpc` resource or the `terraform-aws-module/vpc/aws` community module. + +Nodes need to be able to connect to AWS services plus pull down container images from repos. You can either: +- enable endpoints to the relevant services, if only using ECR repos and for some reason cannot enable public outbound: EC2 API, ECR API, ECR DKR and S3 +- enable outbound public internet access: + - Private subnets: via a NAT gateway or instance + - Public subnets: assign public IPs to nodes + +### `aws-auth` ConfigMap not present + +The module configures the `aws-auth` ConfigMap. This is used by the cluster to grant IAM users RBAC permissions in the cluster. Sometimes the map fails to apply correctly, especially if terraform could not access the cluster endpoint during cluster creation. + +Confirm that the ConfigMap matches the contents of the generated `config-map-aws-auth_${cluster_name}.yaml` file. You can retrieve the live config by running the following in your terraform folder: +`kubectl --kubeconfig=kubeconfig_* -n kube-system get cm aws-auth -o yaml` + +Apply the config with: +`kubectl --kubeconfig=kubeconfig_* apply -f config-map-aws-auth_*.yaml` + +## How can I work with the cluster if I disable the public endpoint? + +You have to interact with the cluster from within the VPC that it's associated with, from an instance that's allowed access via the cluster's security group. + +Creating a new cluster with the public endpoint disabled is harder to achieve. You will either want to pass in a pre-configured cluster security group or apply the `aws-auth` configmap in a separate action. + +## How can I stop Terraform from removing the EKS tags from my VPC and subnets? + +You need to add the tags to the VPC and subnets yourself. See the [basic example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). + +## How do I safely remove old worker groups? + +You've added new worker groups. Deleting worker groups from earlier in the list causes Terraform to want to recreate all worker groups. This is a limitation with how Terraform works and the module using `count` to create the ASGs and other resources. + +The safest and easiest option is to set `asg_min_size` and `asg_max_size` to 0 on the worker groups to "remove". + +## Why does changing the worker group's desired count not do anything? + +The module is configured to ignore this value. Unfortunately Terraform does not support variables within the `lifecycle` block. + +The setting is ignored to allow the cluster autoscaler to work correctly and so that terraform applys do not accidentally remove running workers. + +You can change the desired count via the CLI or console if you're not using the cluster autoscaler. + +If you are not using autoscaling and really want to control the number of nodes via terraform then set the `asg_min_size` and `asg_max_size` instead. AWS will remove a random instance when you scale down. You will have to weigh the risks here. + +## Why are nodes not recreated when the `launch_configuration`/`launch_template` is recreated? + +By default the ASG is not configured to be recreated when the launch configuration or template changes. Terraform spins up new instances and then deletes all the old instances in one go as the AWS provider team have refused to implement rolling updates of autoscaling groups. This is not good for kubernetes stability. + +You need to use a process to drain and cycle the workers. + +You are not using the cluster autoscaler: +- Add a new instance +- Drain an old node `kubectl drain --force --ignore-daemonsets --delete-local-data ip-xxxxxxx.eu-west-1.compute.internal` +- Wait for pods to be Running +- Terminate the old node instance. ASG will start a new instance +- Repeat the drain and delete process until all old nodes are replaced + +You are using the cluster autoscaler: +- Drain an old node `kubectl drain --force --ignore-daemonsets --delete-local-data ip-xxxxxxx.eu-west-1.compute.internal` +- Wait for pods to be Running +- Cluster autoscaler will create new nodes when required +- Repeat until all old nodes are drained +- Cluster autoscaler will terminate the old nodes after 10-60 minutes automatically + +Alternatively you can set the `asg_recreate_on_change = true` worker group option to get the ASG recreated after changes to the launch configuration or template. But be aware of the risks to cluster stability mentioned above. + +## `aws_auth.tf: At 2:14: Unknown token: 2:14 IDENT` + +You are attempting to use a Terraform 0.12 module with Terraform 0.11. + +We highly recommend that you upgrade your EKS Terraform config to 0.12 to take advantage of new features in the module. + +Alternatively you can lock your module to a compatible version if you must stay with terraform 0.11: +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 4.0" + # ... +} +``` From 186c88a34babd7d9ad491f5493059d292cfeb14c Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 26 Sep 2019 19:40:41 +0200 Subject: [PATCH 071/962] Add option to enable lifecycle hooks creation (#532) * add option to enable lifecycle hooks creation * update changelog --- CHANGELOG.md | 2 +- README.md | 1 + variables.tf | 6 ++++++ workers.tf | 6 +++--- workers_launch_template.tf | 6 +++--- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8222332fc3..6cd22ce25a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed - - Write your awesome change here (by @you) + - Add option to enable lifecycle hooks creation (by @barryib) # History diff --git a/README.md b/README.md index 1c338feca1..c79b595113 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | | worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | | worker\_ami\_name\_filter\_prefix | Name prefix filter for AWS EKS worker AMI. Default behaviour will get regular EKS-Optimized AMI but could be set to a EKS-Optimized AMI with GPU Support, e.g. "amazon-eks-gpu-node", or custom AMI | string | `"amazon-eks-node"` | no | +| worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | bool | `"false"` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | diff --git a/variables.tf b/variables.tf index 47d49e79c5..13435c9ad2 100644 --- a/variables.tf +++ b/variables.tf @@ -210,6 +210,12 @@ variable "worker_create_security_group" { default = true } +variable "worker_create_initial_lifecycle_hooks" { + description = "Whether to create initial lifecycle hooks provided in worker groups." + type = bool + default = false +} + variable "permissions_boundary" { description = "If provided, all IAM roles will be created with this permissions boundary attached." type = string diff --git a/workers.tf b/workers.tf index 45db1e449a..5f5ee1d6bd 100644 --- a/workers.tf +++ b/workers.tf @@ -75,10 +75,10 @@ resource "aws_autoscaling_group" "workers" { ) dynamic "initial_lifecycle_hook" { - for_each = lookup(var.worker_groups[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) + for_each = var.worker_create_initial_lifecycle_hooks ? lookup(var.worker_groups[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) : [] content { - name = lookup(initial_lifecycle_hook.value, "name", null) - lifecycle_transition = lookup(initial_lifecycle_hook.value, "lifecycle_transition", null) + name = initial_lifecycle_hook.value["name"] + lifecycle_transition = initial_lifecycle_hook.value["lifecycle_transition"] notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 258d536f09..e44a909401 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -151,10 +151,10 @@ resource "aws_autoscaling_group" "workers_launch_template" { } dynamic "initial_lifecycle_hook" { - for_each = lookup(var.worker_groups_launch_template[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) + for_each = var.worker_create_initial_lifecycle_hooks ? lookup(var.worker_groups_launch_template[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) : [] content { - name = lookup(initial_lifecycle_hook.value, "name", null) - lifecycle_transition = lookup(initial_lifecycle_hook.value, "lifecycle_transition", null) + name = initial_lifecycle_hook.value["name"] + lifecycle_transition = initial_lifecycle_hook.value["lifecycle_transition"] notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) From f3beb8e5fc22bf3384cc3f47a326ebca7f67516a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Gir=C3=B3n?= Date: Fri, 27 Sep 2019 03:28:55 -0400 Subject: [PATCH 072/962] =?UTF-8?q?Update=20comments=20for=20'asg=5Frecrea?= =?UTF-8?q?te=5Fon=5Fchange'=20attribute=20in=20'workers=5Fgr=E2=80=A6=20(?= =?UTF-8?q?#536)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update comments for 'asg_recreate_on_change' attribute in 'workers_group_defaults_defaults' for better clarify of what resources it's talking about. * Expand a few more acronyms and abbreviations. --- local.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/local.tf b/local.tf index c388221961..a658cbc819 100644 --- a/local.tf +++ b/local.tf @@ -21,14 +21,14 @@ locals { workers_group_defaults_defaults = { name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. - tags = [] # A list of map defining extra tags to be applied to the worker group ASG. + tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. ami_id = data.aws_ami.eks_worker.id # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI. asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. asg_max_size = "3" # Maximum worker capacity in the autoscaling group. asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. asg_force_delete = false # Enable forced deletion for the autoscaling group. asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. - asg_recreate_on_change = false # Recreate the autoscaling group when LT or LC change. + asg_recreate_on_change = false # Recreate the autoscaling group when the Launch Template or Launch Configuration change. instance_type = "m4.large" # Size of the workers instances. spot_price = "" # Cost of spot instance. placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". @@ -37,7 +37,7 @@ locals { root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". key_name = "" # The key name that should be used for the instances in the autoscaling group pre_userdata = "" # userdata to pre-append to the default userdata. - bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI. + bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI (Amazon Machine Image). additional_userdata = "" # userdata to append to the default userdata. ebs_optimized = true # sets whether to use ebs optimization on supported types. enable_monitoring = true # Enables/disables detailed monitoring. @@ -50,7 +50,7 @@ locals { iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. iam_role_id = "local.default_iam_role_id" # A custom IAM role id. Incompatible with iam_instance_profile_name. Literal local.default_iam_role_id will never be used but if iam_role_id is not set, the local.default_iam_role_id interpolation will be used. suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] - target_group_arns = null # A list of ALB target group ARNs to be associated to the ASG + target_group_arns = null # A list of Application LoadBalancer (ALB) target group ARNs to be associated to the autoscaling group enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] placement_group = "" # The name of the placement group into which to launch the instances, if any. service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS @@ -62,7 +62,7 @@ locals { launch_template_placement_tenancy = "default" # The placement tenancy for instances launch_template_placement_group = "" # The name of the placement group into which to launch the instances, if any. root_encrypted = "" # Whether the volume should be encrypted or not - eni_delete = true # Delete the ENI on termination (if set to false you will have to manually delete before destroying) + eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs market_type = null # Settings for launch templates with mixed instances policy From 6bd020a041a65e9d2a5881037c45f2f3f87f11f6 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Fri, 27 Sep 2019 12:08:05 +0200 Subject: [PATCH 073/962] Expand and clarify the networking section. (#538) --- docs/faq.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 8aa2511b0d..089ce3485e 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -14,15 +14,17 @@ Often caused by a networking or endpoint configuration issue. At least one of the cluster public or private endpoints must be enabled in order for access to the cluster to work. -Your nodes need to be able to contact the EKS cluster endpoint. By default the module only creates a public endpoint. You should also enable the private endpoint by setting `cluster_endpoint_private_access = true` on this module. +Nodes need to be able to contact the EKS cluster endpoint. By default the module only creates a public endpoint. To access this endpoint the nodes need outgoing internet access: +- Nodes in private subnets: via a NAT gateway or instance. This will need adding along with appropriate routing rules. +- Nodes in public subnets: assign public IPs to nodes. Set `public_ip = true` in the `worker_groups` list on this module. -If you have the private endpoint enabled ensure that you also have VPC DNS enabled. Set `enable_dns_hostnames = true` on your `aws_vpc` resource or the `terraform-aws-module/vpc/aws` community module. +Cluster private endpoint can also be enabled by setting `cluster_endpoint_private_access = true` on this module. Node calls to the endpoint stay within the VPC. -Nodes need to be able to connect to AWS services plus pull down container images from repos. You can either: -- enable endpoints to the relevant services, if only using ECR repos and for some reason cannot enable public outbound: EC2 API, ECR API, ECR DKR and S3 -- enable outbound public internet access: - - Private subnets: via a NAT gateway or instance - - Public subnets: assign public IPs to nodes +When the private endpoint is enabled ensure that VPC DNS resolution and hostnames are also enabled: +- If managing the VPC with Terraform: set `enable_dns_hostnames = true` and `enable_dns_support = true` on the `aws_vpc` resource. The [`terraform-aws-module/vpc/aws`](https://github.com/terraform-aws-modules/terraform-aws-vpc/) community module also has these variables. +- Otherwise refer to the [AWS VPC docs](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html#vpc-dns-updating) and [AWS EKS Cluster Endpoint Access docs](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html) for more information. + +Nodes need to be able to connect to other AWS services plus pull down container images from repos. If for some reason you cannot enable public internet access for nodes you can add VPC endpoints to the relevant services: EC2 API, ECR API, ECR DKR and S3. ### `aws-auth` ConfigMap not present From 18748a5f3878da54af13cfd5d37ae28232cbda36 Mon Sep 17 00:00:00 2001 From: Wi1dcard Date: Fri, 27 Sep 2019 19:18:31 +0800 Subject: [PATCH 074/962] Update autoscaling.md (#531) --- CHANGELOG.md | 1 + docs/autoscaling.md | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cd22ce25a..5fedef1bbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed - Add option to enable lifecycle hooks creation (by @barryib) + - Remove helm chart value `sslCertPath` described in `docs/autoscaling.md` (by @wi1dcard) # History diff --git a/docs/autoscaling.md b/docs/autoscaling.md index c4b6d98184..2aabab83d2 100644 --- a/docs/autoscaling.md +++ b/docs/autoscaling.md @@ -12,8 +12,6 @@ The [helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autos rbac: create: true -sslCertPath: /etc/ssl/certs/ca-bundle.crt - cloudProvider: aws awsRegion: YOUR_AWS_REGION @@ -34,4 +32,4 @@ There is a variable `asg_desired_capacity` given in the `local.tf` file, current ## See More -[Using AutoScalingGroup MixedInstancesPolicy](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-autoscalinggroup-mixedinstancespolicy) \ No newline at end of file +[Using AutoScalingGroup MixedInstancesPolicy](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-autoscalinggroup-mixedinstancespolicy) From f98518f050e0f2206f7f46ad84469b12154a37f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=9B=B8=E8=BD=A9?= <9570112+nauxliu@users.noreply.github.com> Date: Tue, 1 Oct 2019 01:03:52 +0800 Subject: [PATCH 075/962] Update badges (#542) --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c79b595113..9d2b3d2d30 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ -# terraform-aws-eks +# terraform-aws-eks + +[![Lint Status](https://github.com/terraform-aws-modules/terraform-aws-eks/workflows/Lint/badge.svg)](https://github.com/terraform-aws-modules/terraform-aws-eks/actions) +[![LICENSE](https://img.shields.io/github/license/terraform-aws-modules/terraform-aws-eks)](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/LICENSE) A terraform module to create a managed Kubernetes cluster on AWS EKS. Available through the [Terraform registry](https://registry.terraform.io/modules/terraform-aws-modules/eks/aws). @@ -6,10 +9,6 @@ Inspired by and adapted from [this doc](https://www.terraform.io/docs/providers/ and its [source code](https://github.com/terraform-providers/terraform-provider-aws/tree/master/examples/eks-getting-started). Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html). -| Branch | Build status | -| ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| master | [![build Status](https://travis-ci.org/terraform-aws-modules/terraform-aws-eks.svg?branch=master)](https://travis-ci.org/terraform-aws-modules/terraform-aws-eks) | - ## Assumptions * You want to create an EKS cluster and an autoscaling group of workers for the cluster. From 09635a36b771a301e7c60590c5175849be253b2c Mon Sep 17 00:00:00 2001 From: Harshal Shah Date: Fri, 4 Oct 2019 11:57:37 +0200 Subject: [PATCH 076/962] remove the launch template mixed variable from examples (#545) * Add disclaimer for launch template mixed Or maybe it is better to remove this variable altogether. * Removing the mixed variable altogether And updating the existing variable `worker_groups_launch_template` --- docs/spot-instances.md | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/docs/spot-instances.md b/docs/spot-instances.md index 33bd807ff1..4dcd8f1e85 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -73,28 +73,17 @@ Launch Template support is a recent addition to both AWS and this module. It mig } ] - worker_groups_launch_template_mixed = [ - { - name = "spot-1" - override_instance_types = ["m5.large", "c5.large", "t3.large", "r5.large"] - spot_instance_pools = 3 - asg_max_size = 5 - asg_desired_size = 5 - autoscaling_enabled = true - kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" - } - ] worker_groups_launch_template = [ { - name = "spot-2" - instance_type = "m4.xlarge" - asg_max_size = 5 - asg_desired_size = 5 - autoscaling_enabled = true - kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" - market_type = "spot" - } + name = "spot-1" + override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] + spot_instance_pools = 4 + asg_max_size = 5 + asg_desired_capacity = 5 + kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" + public_ip = true + }, ] ``` From 813c607e908ec043e004f08cdb4c71e9ee5f17fb Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Fri, 4 Oct 2019 14:25:19 +0200 Subject: [PATCH 077/962] Make "dangerous" policy attachments optional (#539) --- CHANGELOG.md | 1 + README.md | 5 +++++ outputs.tf | 9 +++++++++ variables.tf | 18 ++++++++++++++++++ workers.tf | 6 +++--- 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fedef1bbe..5a75bf0b20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Add option to enable lifecycle hooks creation (by @barryib) - Remove helm chart value `sslCertPath` described in `docs/autoscaling.md` (by @wi1dcard) + - Attaching of IAM policies for autoscaler and CNI to the worker nodes now optional (by @dpiddockcmp) # History diff --git a/README.md b/README.md index 9d2b3d2d30..ed0eef59d7 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| attach\_worker\_autoscaling\_policy | Whether to attach the module managed cluster autoscaling iam policy to the default worker IAM role. This requires `manage_worker_autoscaling_policy = true` | bool | `"true"` | no | +| attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | bool | `"true"` | no | | cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | bool | `"true"` | no | | cluster\_create\_timeout | Timeout value when creating the EKS cluster. | string | `"15m"` | no | | cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | string | `"15m"` | no | @@ -130,6 +132,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | local\_exec\_interpreter | Command to run for local-exec resources. Must be a shell-style interpreter. If you are on Windows Git Bash is a good choice. | list(string) | `[ "/bin/sh", "-c" ]` | no | | manage\_aws\_auth | Whether to apply the aws-auth configmap file. | string | `"true"` | no | | manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | bool | `"true"` | no | +| manage\_worker\_autoscaling\_policy | Whether to let the module manage the cluster autoscaling iam policy. | bool | `"true"` | no | | manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | bool | `"true"` | no | | map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | | map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | @@ -170,6 +173,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | +| worker\_autoscaling\_policy\_arn | ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | +| worker\_autoscaling\_policy\_name | Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | | worker\_iam\_instance\_profile\_arns | default IAM instance profile ARN for EKS worker groups | | worker\_iam\_instance\_profile\_names | default IAM instance profile name for EKS worker groups | | worker\_iam\_role\_arn | default IAM role ARN for EKS worker groups | diff --git a/outputs.tf b/outputs.tf index 3a69d502b3..a195b21fcf 100644 --- a/outputs.tf +++ b/outputs.tf @@ -142,3 +142,12 @@ output "worker_iam_role_arn" { )[0] } +output "worker_autoscaling_policy_name" { + description = "Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true`" + value = concat(aws_iam_policy.worker_autoscaling[*].name, [""])[0] +} + +output "worker_autoscaling_policy_arn" { + description = "ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true`" + value = concat(aws_iam_policy.worker_autoscaling[*].arn, [""])[0] +} diff --git a/variables.tf b/variables.tf index 13435c9ad2..b3ca3c264c 100644 --- a/variables.tf +++ b/variables.tf @@ -263,3 +263,21 @@ variable "workers_role_name" { type = string default = "" } + +variable "manage_worker_autoscaling_policy" { + description = "Whether to let the module manage the cluster autoscaling iam policy." + type = bool + default = true +} + +variable "attach_worker_autoscaling_policy" { + description = "Whether to attach the module managed cluster autoscaling iam policy to the default worker IAM role. This requires `manage_worker_autoscaling_policy = true`" + type = bool + default = true +} + +variable "attach_worker_cni_policy" { + description = "Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster." + type = bool + default = true +} diff --git a/workers.tf b/workers.tf index 5f5ee1d6bd..0f2effa564 100644 --- a/workers.tf +++ b/workers.tf @@ -342,7 +342,7 @@ resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { } resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { - count = var.manage_worker_iam_resources ? 1 : 0 + count = var.manage_worker_iam_resources && var.attach_worker_cni_policy ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" role = aws_iam_role.workers[0].name } @@ -360,13 +360,13 @@ resource "aws_iam_role_policy_attachment" "workers_additional_policies" { } resource "aws_iam_role_policy_attachment" "workers_autoscaling" { - count = var.manage_worker_iam_resources ? 1 : 0 + count = var.manage_worker_iam_resources && var.manage_worker_autoscaling_policy && var.attach_worker_autoscaling_policy ? 1 : 0 policy_arn = aws_iam_policy.worker_autoscaling[0].arn role = aws_iam_role.workers[0].name } resource "aws_iam_policy" "worker_autoscaling" { - count = var.manage_worker_iam_resources ? 1 : 0 + count = var.manage_worker_iam_resources && var.manage_worker_autoscaling_policy ? 1 : 0 name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this.name}" description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this.name}" policy = data.aws_iam_policy_document.worker_autoscaling.json From edcbba64ef46c615c5da97e67deadf90d75c1553 Mon Sep 17 00:00:00 2001 From: Morgan Christiansson Date: Fri, 4 Oct 2019 13:48:53 +0100 Subject: [PATCH 078/962] Add tags to aws_eks_cluster introduced in terraform-provider-aws 2.31.0 (#550) - https://github.com/terraform-providers/terraform-provider-aws/pull/10307 - https://aws.amazon.com/about-aws/whats-new/2019/09/amazon-eks-supports-cluster-tagging/ --- CHANGELOG.md | 1 + cluster.tf | 1 + versions.tf | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a75bf0b20..aa56a1563f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added +- Added `tags` to `aws_eks_cluster` introduced by terraform-provider-aws 2.31.0 (by @morganchristiansson) - Write your awesome addition here (by @you) ### Changed diff --git a/cluster.tf b/cluster.tf index 7fa9409f33..ba33c796e4 100644 --- a/cluster.tf +++ b/cluster.tf @@ -11,6 +11,7 @@ resource "aws_eks_cluster" "this" { enabled_cluster_log_types = var.cluster_enabled_log_types role_arn = local.cluster_iam_role_arn version = var.cluster_version + tags = var.tags vpc_config { security_group_ids = [local.cluster_security_group_id] diff --git a/versions.tf b/versions.tf index 5e2d714efc..f6be617083 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.12" required_providers { - aws = ">= 2.28.1" + aws = ">= 2.31.0" local = ">= 1.2" null = ">= 2.1" template = ">= 2.1" From 1be1a02e23a47a0deecf4460c9e3783bb6d13252 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 7 Oct 2019 20:53:31 +0200 Subject: [PATCH 079/962] New release 6.0.2 (#553) --- CHANGELOG.md | 19 +++++++++++++------ version | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa56a1563f..147223e9fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,21 +7,30 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v6.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...HEAD)] - 2019-10-??] +## [[v6.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...HEAD)] - 2019-10-??] ### Added -- Added `tags` to `aws_eks_cluster` introduced by terraform-provider-aws 2.31.0 (by @morganchristiansson) - Write your awesome addition here (by @you) +### Changed + + - Write your awesome change here (by @you) + +# History + +## [[v6.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2)] - 2019-10-07] + +### Added + +- Added `tags` to `aws_eks_cluster` introduced by terraform-provider-aws 2.31.0 (by @morganchristiansson) + ### Changed - Add option to enable lifecycle hooks creation (by @barryib) - Remove helm chart value `sslCertPath` described in `docs/autoscaling.md` (by @wi1dcard) - Attaching of IAM policies for autoscaler and CNI to the worker nodes now optional (by @dpiddockcmp) -# History - ## [[v6.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...v6.0.1)] - 2019-09-25] ### Added @@ -33,8 +42,6 @@ project adheres to [Semantic Versioning](http://semver.org/). - Use null as default value for `target_group_arns` attribute of worker autoscaling group (by @tatusl) - Output empty string when cluster identity is empty (by @tbarry) -# History - ## [[v6.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v6.0.0)] - 2019-09-17] - Added `market_type` to `workers_launch_template.tf` allow the usage of spot nodegroups without mixed instances policy. diff --git a/version b/version index 23e595e7bd..860cf3e47f 100644 --- a/version +++ b/version @@ -1 +1 @@ -v6.0.1 +v6.0.2 From 2e2f2acadad7b2df84cc110708be73155d4b0f84 Mon Sep 17 00:00:00 2001 From: Jeff Hastings Date: Mon, 21 Oct 2019 06:57:29 -0400 Subject: [PATCH 080/962] change ebs_optimized mapping to list where not supported (#561) * change ebs_optimized mapping to list where not supported * update changelog --- CHANGELOG.md | 2 +- local.tf | 188 ++++++++----------------------------- workers.tf | 11 +-- workers_launch_template.tf | 10 +- 4 files changed, 48 insertions(+), 163 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 147223e9fd..6bf8de0047 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed - - Write your awesome change here (by @you) + - Changed logic for setting default ebs_optimized to only require maintaining a list of instance types that don't support it (by @jeffmhastings) # History diff --git a/local.tf b/local.tf index a658cbc819..9bf6d18dec 100644 --- a/local.tf +++ b/local.tf @@ -80,155 +80,41 @@ locals { var.workers_group_defaults, ) - ebs_optimized = { - "c1.medium" = false - "c1.xlarge" = true - "c3.large" = false - "c3.xlarge" = true - "c3.2xlarge" = true - "c3.4xlarge" = true - "c3.8xlarge" = false - "c4.large" = true - "c4.xlarge" = true - "c4.2xlarge" = true - "c4.4xlarge" = true - "c4.8xlarge" = true - "c5.large" = true - "c5.xlarge" = true - "c5.2xlarge" = true - "c5.4xlarge" = true - "c5.9xlarge" = true - "c5.18xlarge" = true - "c5d.large" = true - "c5d.xlarge" = true - "c5d.2xlarge" = true - "c5d.4xlarge" = true - "c5d.9xlarge" = true - "c5d.18xlarge" = true - "cc2.8xlarge" = false - "cr1.8xlarge" = false - "d2.xlarge" = true - "d2.2xlarge" = true - "d2.4xlarge" = true - "d2.8xlarge" = true - "f1.2xlarge" = true - "f1.4xlarge" = true - "f1.16xlarge" = true - "g2.2xlarge" = true - "g2.8xlarge" = false - "g3s.xlarge" = true - "g3.4xlarge" = true - "g3.8xlarge" = true - "g3.16xlarge" = true - "h1.2xlarge" = true - "h1.4xlarge" = true - "h1.8xlarge" = true - "h1.16xlarge" = true - "hs1.8xlarge" = false - "i2.xlarge" = true - "i2.2xlarge" = true - "i2.4xlarge" = true - "i2.8xlarge" = false - "i3.large" = true - "i3.xlarge" = true - "i3.2xlarge" = true - "i3.4xlarge" = true - "i3.8xlarge" = true - "i3.16xlarge" = true - "i3.metal" = true - "m1.small" = false - "m1.medium" = false - "m1.large" = true - "m1.xlarge" = true - "m2.xlarge" = false - "m2.2xlarge" = true - "m2.4xlarge" = true - "m3.medium" = false - "m3.large" = false - "m3.xlarge" = true - "m3.2xlarge" = true - "m4.large" = true - "m4.xlarge" = true - "m4.2xlarge" = true - "m4.4xlarge" = true - "m4.10xlarge" = true - "m4.16xlarge" = true - "m5a.large" = true - "m5a.xlarge" = true - "m5a.2xlarge" = true - "m5a.4xlarge" = true - "m5a.12xlarge" = true - "m5a.24xlarge" = true - "m5.large" = true - "m5.xlarge" = true - "m5.2xlarge" = true - "m5.4xlarge" = true - "m5.12xlarge" = true - "m5.24xlarge" = true - "m5d.large" = true - "m5d.xlarge" = true - "m5d.2xlarge" = true - "m5d.4xlarge" = true - "m5d.12xlarge" = true - "m5d.24xlarge" = true - "p2.xlarge" = true - "p2.8xlarge" = true - "p2.16xlarge" = true - "p3.2xlarge" = true - "p3.8xlarge" = true - "p3.16xlarge" = true - "r3.large" = false - "r3.xlarge" = true - "r3.2xlarge" = true - "r3.4xlarge" = true - "r3.8xlarge" = false - "r4.large" = true - "r4.xlarge" = true - "r4.2xlarge" = true - "r4.4xlarge" = true - "r4.8xlarge" = true - "r4.16xlarge" = true - "r5a.large" = true - "r5a.xlarge" = true - "r5a.2xlarge" = true - "r5a.4xlarge" = true - "r5a.12xlarge" = true - "r5a.24xlarge" = true - "r5.large" = true - "r5.xlarge" = true - "r5.2xlarge" = true - "r5.4xlarge" = true - "r5.12xlarge" = true - "r5.24xlarge" = true - "t1.micro" = false - "t2.nano" = false - "t2.micro" = false - "t2.small" = false - "t2.medium" = false - "t2.large" = false - "t2.xlarge" = false - "t2.2xlarge" = false - "t3a.nano" = true - "t3a.micro" = true - "t3a.small" = true - "t3a.medium" = true - "t3a.large" = true - "t3a.xlarge" = true - "t3a.2xlarge" = true - "t3.nano" = true - "t3.micro" = true - "t3.small" = true - "t3.medium" = true - "t3.large" = true - "t3.xlarge" = true - "t3.2xlarge" = true - "x1.16xlarge" = true - "x1.32xlarge" = true - "x1e.xlarge" = true - "x1e.2xlarge" = true - "x1e.4xlarge" = true - "x1e.8xlarge" = true - "x1e.16xlarge" = true - "x1e.32xlarge" = true - } + ebs_optimized_not_supported = [ + "c1.medium", + "c3.8xlarge", + "c3.large", + "c5d.12xlarge", + "c5d.24xlarge", + "c5d.metal", + "cc2.8xlarge", + "cr1.8xlarge", + "g2.8xlarge", + "g4dn.metal", + "hs1.8xlarge", + "i2.8xlarge", + "m1.medium", + "m1.small", + "m2.xlarge", + "m3.large", + "m3.medium", + "m5ad.16xlarge", + "m5ad.8xlarge", + "m5dn.metal", + "m5n.metal", + "r3.8xlarge", + "r3.large", + "r5ad.16xlarge", + "r5ad.8xlarge", + "r5dn.metal", + "r5n.metal", + "t1.micro", + "t2.2xlarge", + "t2.large", + "t2.medium", + "t2.micro", + "t2.nano", + "t2.small", + "t2.xlarge" + ] } diff --git a/workers.tf b/workers.tf index 0f2effa564..ca0bd728d6 100644 --- a/workers.tf +++ b/workers.tf @@ -182,15 +182,14 @@ resource "aws_launch_configuration" "workers" { ebs_optimized = lookup( var.worker_groups[count.index], "ebs_optimized", - lookup( - local.ebs_optimized, + ! contains( + local.ebs_optimized_not_supported, lookup( var.worker_groups[count.index], "instance_type", - local.workers_group_defaults["instance_type"], - ), - false, - ), + local.workers_group_defaults["instance_type"] + ) + ) ) enable_monitoring = lookup( var.worker_groups[count.index], diff --git a/workers_launch_template.tf b/workers_launch_template.tf index e44a909401..6bb6437b67 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -272,18 +272,18 @@ resource "aws_launch_template" "workers_launch_template" { user_data = base64encode( data.template_file.launch_template_userdata.*.rendered[count.index], ) + ebs_optimized = lookup( var.worker_groups_launch_template[count.index], "ebs_optimized", - lookup( - local.ebs_optimized, + ! contains( + local.ebs_optimized_not_supported, lookup( var.worker_groups_launch_template[count.index], "instance_type", local.workers_group_defaults["instance_type"], - ), - false, - ), + ) + ) ) credit_specification { From c81e1d2fa75a2ddd50daa0bacac4548f6f4a18bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20M=2E=20Cust=C3=B3dio?= Date: Mon, 21 Oct 2019 12:04:04 +0100 Subject: [PATCH 081/962] Allow for specifying a custom AMI for the worker nodes. (#551) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bruno Miguel Custódio --- CHANGELOG.md | 1 + README.md | 4 ++-- data.tf | 9 ++++++--- variables.tf | 10 +++++----- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bf8de0047..9cf41f988c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added +- **Breaking:** Allow for specifying a custom AMI for the worker nodes. (by @bmcstdio) - Write your awesome addition here (by @you) ### Changed diff --git a/README.md b/README.md index ed0eef59d7..4f475d7c03 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | tags | A map of tags to add to all resources. | map(string) | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | -| worker\_ami\_name\_filter | Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. "v20190220" | string | `"v*"` | no | -| worker\_ami\_name\_filter\_prefix | Name prefix filter for AWS EKS worker AMI. Default behaviour will get regular EKS-Optimized AMI but could be set to a EKS-Optimized AMI with GPU Support, e.g. "amazon-eks-gpu-node", or custom AMI | string | `"amazon-eks-node"` | no | +| worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | +| worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"602401143452"` | no | | worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | bool | `"false"` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | diff --git a/data.tf b/data.tf index 3a5ae4fcb4..57bcad78d8 100644 --- a/data.tf +++ b/data.tf @@ -1,3 +1,7 @@ +locals { + worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" +} + data "aws_region" "current" { } @@ -19,13 +23,12 @@ data "aws_iam_policy_document" "workers_assume_role_policy" { data "aws_ami" "eks_worker" { filter { name = "name" - values = ["${var.worker_ami_name_filter_prefix}-${var.cluster_version}-${var.worker_ami_name_filter}"] + values = [local.worker_ami_name_filter] } most_recent = true - # Owner ID of AWS EKS team - owners = ["602401143452"] + owners = [var.worker_ami_owner_id] } data "aws_iam_policy_document" "cluster_assume_role_policy" { diff --git a/variables.tf b/variables.tf index b3ca3c264c..9d3a4796be 100644 --- a/variables.tf +++ b/variables.tf @@ -121,15 +121,15 @@ variable "worker_security_group_id" { } variable "worker_ami_name_filter" { - description = "Additional name filter for AWS EKS worker AMI. Default behaviour will get latest for the cluster_version but could be set to a release from amazon-eks-ami, e.g. \"v20190220\"" + description = "Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used." type = string - default = "v*" + default = "" } -variable "worker_ami_name_filter_prefix" { - description = "Name prefix filter for AWS EKS worker AMI. Default behaviour will get regular EKS-Optimized AMI but could be set to a EKS-Optimized AMI with GPU Support, e.g. \"amazon-eks-gpu-node\", or custom AMI" +variable "worker_ami_owner_id" { + description = "The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft')." type = string - default = "amazon-eks-node" + default = "602401143452" // The ID of the owner of the official AWS EKS AMIs. } variable "worker_additional_security_group_ids" { From 4369f0271b12cd3f98f705df036662f0660efa4b Mon Sep 17 00:00:00 2001 From: Joshua Spence Date: Mon, 21 Oct 2019 23:55:29 +1100 Subject: [PATCH 082/962] Allow `config_output_path` to specify the full path (#549) --- CHANGELOG.md | 3 ++- README.md | 4 ++-- kubectl.tf | 2 +- variables.tf | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cf41f988c..046e855c82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,8 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed - - Changed logic for setting default ebs_optimized to only require maintaining a list of instance types that don't support it (by @jeffmhastings) +- **Breaking:** The `kubectl` configuration file can now be fully-specified using `config_output_path`. Previously it was assumed that `config_output_path` referred to a directory and always ended with a forward slash. This is a breaking change if `config_output_path` does **not** end with a forward slash (which was advised against by the documentation). +- Changed logic for setting default ebs_optimized to only require maintaining a list of instance types that don't support it (by @jeffmhastings) # History diff --git a/README.md b/README.md index 4f475d7c03..f2fc94cf91 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers | string | `""` | no | | cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.14"` | no | -| config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Should end in a forward slash `/` . | string | `"./"` | no | +| config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | string | `"./"` | no | | iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | | kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | | kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | string | `"aws-iam-authenticator"` | no | @@ -188,4 +188,4 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | workers\_launch\_template\_latest\_versions | Latest versions of the worker launch templates. | | workers\_user\_data | User data of worker groups | - \ No newline at end of file + diff --git a/kubectl.tf b/kubectl.tf index ede1575cae..5a70828f0d 100644 --- a/kubectl.tf +++ b/kubectl.tf @@ -1,6 +1,6 @@ resource "local_file" "kubeconfig" { count = var.write_kubeconfig ? 1 : 0 content = data.template_file.kubeconfig.rendered - filename = "${var.config_output_path}kubeconfig_${var.cluster_name}" + filename = "${substr(var.config_output_path, -1, 1) == "/" ? "${var.config_output_path}kubeconfig_${var.cluster_name}" : var.config_output_path}" } diff --git a/variables.tf b/variables.tf index 9d3a4796be..a4d2d4dc10 100644 --- a/variables.tf +++ b/variables.tf @@ -32,7 +32,7 @@ variable "cluster_version" { } variable "config_output_path" { - description = "Where to save the Kubectl config file (if `write_kubeconfig = true`). Should end in a forward slash `/` ." + description = "Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`." type = string default = "./" } From 3f346d16109c4bc53fe256fcaae6bc08613d1173 Mon Sep 17 00:00:00 2001 From: Miguel Ferreira Date: Thu, 24 Oct 2019 18:13:06 +0200 Subject: [PATCH 083/962] Remove region from vars mapping for kubeconfig (#556) The template file data source was injecting a region variable that is not defined in the template. Once removed the variable the aws region data source was not used anywhere else, so it is removed as well. --- data.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/data.tf b/data.tf index 57bcad78d8..1a28e25412 100644 --- a/data.tf +++ b/data.tf @@ -2,9 +2,6 @@ locals { worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" } -data "aws_region" "current" { -} - data "aws_iam_policy_document" "workers_assume_role_policy" { statement { sid = "EKSWorkerAssumeRole" @@ -52,7 +49,6 @@ data "template_file" "kubeconfig" { vars = { kubeconfig_name = local.kubeconfig_name endpoint = aws_eks_cluster.this.endpoint - region = data.aws_region.current.name cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data aws_authenticator_command = var.kubeconfig_aws_authenticator_command aws_authenticator_command_args = length(var.kubeconfig_aws_authenticator_command_args) > 0 ? " - ${join( From c90b33780c37cd77eea1a9a1714d0f7c059ec179 Mon Sep 17 00:00:00 2001 From: Arnaud Venturi Date: Fri, 25 Oct 2019 09:14:01 +0000 Subject: [PATCH 084/962] Increase minimum terraform version to include yamlencode (#570) * Increase minimum terraform version to include yamlencode https://github.com/hashicorp/terraform/blob/master/CHANGELOG.md#0122-june-12-2019 * Update Changelog --- CHANGELOG.md | 1 + examples/launch_templates/main.tf | 2 +- examples/spot_instances/main.tf | 2 +- versions.tf | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 046e855c82..4111f0cf98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - **Breaking:** The `kubectl` configuration file can now be fully-specified using `config_output_path`. Previously it was assumed that `config_output_path` referred to a directory and always ended with a forward slash. This is a breaking change if `config_output_path` does **not** end with a forward slash (which was advised against by the documentation). - Changed logic for setting default ebs_optimized to only require maintaining a list of instance types that don't support it (by @jeffmhastings) +- Bumped minimum terraform version to 0.12.2 to prevent an error on yamlencode function (by @toadjaune) # History diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 008a4f32df..da2589edc7 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.12.0" + required_version = ">= 0.12.2" } provider "aws" { diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 741531d41c..6c5cb47896 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.12.0" + required_version = ">= 0.12.2" } provider "aws" { diff --git a/versions.tf b/versions.tf index f6be617083..0a6046ef7f 100644 --- a/versions.tf +++ b/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.2" required_providers { aws = ">= 2.31.0" From be6fa61d0d63c19397aff894baee787912844c04 Mon Sep 17 00:00:00 2001 From: chenrui Date: Fri, 25 Oct 2019 16:42:32 -0400 Subject: [PATCH 085/962] Add `terraform_validate` and `terraform_tflint` pre-commit hooks (#571) --- .pre-commit-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac4b62b453..f21c517082 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,7 @@ - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.16.0 + rev: v1.19.0 hooks: - id: terraform_fmt - id: terraform_docs + - id: terraform_validate + - id: terraform_tflint From 2d52e06786cdb9feb742b7fc1968ffa5b87cf255 Mon Sep 17 00:00:00 2001 From: Joonsung Lee Date: Mon, 28 Oct 2019 22:16:45 +0900 Subject: [PATCH 086/962] Add Windows support (#555) * Add Windows support * Assign eks:kube-proxy-windows group to worker nodes * Add Instructions for adding Windows Workers at FAQ.md * Remove unnecessary variables from userdata_windows.tpl * Update CHANGELOG.md --- CHANGELOG.md | 1 + README.md | 2 ++ aws_auth.tf | 10 +++++++++ data.tf | 37 +++++++++++++++++++++++++++++----- docs/faq.md | 6 ++++++ local.tf | 6 +++++- templates/userdata_windows.tpl | 11 ++++++++++ templates/worker-role.tpl | 3 +++ variables.tf | 12 +++++++++++ workers.tf | 2 +- workers_launch_template.tf | 2 +- 11 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 templates/userdata_windows.tpl diff --git a/CHANGELOG.md b/CHANGELOG.md index 4111f0cf98..fb343ca936 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added - **Breaking:** Allow for specifying a custom AMI for the worker nodes. (by @bmcstdio) +- Added support for Windows workers AMIs (by @hodduc) - Write your awesome addition here (by @you) ### Changed diff --git a/README.md b/README.md index f2fc94cf91..edcc733ced 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,9 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | | worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | +| worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | | worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"602401143452"` | no | +| worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"801119661308"` | no | | worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | bool | `"false"` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | diff --git a/aws_auth.tf b/aws_auth.tf index d6531107cb..bd3221194d 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -47,6 +47,11 @@ data "template_file" "launch_template_worker_role_arns" { ), count.index, )}" + platform = lookup( + var.worker_groups_launch_template[count.index], + "platform", + local.workers_group_defaults["platform"] + ) } } @@ -63,6 +68,11 @@ data "template_file" "worker_role_arns" { ), count.index, )}" + platform = lookup( + var.worker_groups[count.index], + "platform", + local.workers_group_defaults["platform"] + ) } } diff --git a/data.tf b/data.tf index 1a28e25412..8fee32f9cf 100644 --- a/data.tf +++ b/data.tf @@ -1,5 +1,6 @@ locals { - worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" + worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" + worker_ami_name_filter_windows = var.worker_ami_name_filter_windows != "" ? var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${var.cluster_version}-*" } data "aws_iam_policy_document" "workers_assume_role_policy" { @@ -28,6 +29,24 @@ data "aws_ami" "eks_worker" { owners = [var.worker_ami_owner_id] } +data "aws_ami" "eks_worker_windows" { + filter { + name = "name" + values = [local.worker_ami_name_filter_windows] + } + + filter { + name = "platform" + values = ["windows"] + } + + most_recent = true + + # Owner ID of AWS EKS team (windows) + owners = [var.worker_ami_owner_id_windows] +} + + data "aws_iam_policy_document" "cluster_assume_role_policy" { statement { sid = "EKSClusterAssumeRole" @@ -85,8 +104,12 @@ EOF } data "template_file" "userdata" { - count = local.worker_group_count - template = file("${path.module}/templates/userdata.sh.tpl") + count = local.worker_group_count + template = file( + lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? + "${path.module}/templates/userdata_windows.tpl" : + "${path.module}/templates/userdata.sh.tpl" + ) vars = { cluster_name = aws_eks_cluster.this.name @@ -116,8 +139,12 @@ data "template_file" "userdata" { } data "template_file" "launch_template_userdata" { - count = local.worker_group_launch_template_count - template = file("${path.module}/templates/userdata.sh.tpl") + count = local.worker_group_launch_template_count + template = file( + lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? + "${path.module}/templates/userdata_windows.tpl" : + "${path.module}/templates/userdata.sh.tpl" + ) vars = { cluster_name = aws_eks_cluster.this.name diff --git a/docs/faq.md b/docs/faq.md index 089ce3485e..d793f28fea 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -98,3 +98,9 @@ module "eks" { # ... } ``` + +## How can I use Windows workers? + +To enable Windows support for your EKS cluster, you should apply some configs manually. See the [Enabling Windows Support (Windows/MacOS/Linux)](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support). + +Windows worker nodes requires additional cluster role (eks:kube-proxy-windows). If you are adding windows workers to existing cluster, you should apply config-map-aws-auth again. diff --git a/local.tf b/local.tf index 9bf6d18dec..7a4b457007 100644 --- a/local.tf +++ b/local.tf @@ -19,10 +19,13 @@ locals { worker_group_count = length(var.worker_groups) worker_group_launch_template_count = length(var.worker_groups_launch_template) + default_ami_id_linux = data.aws_ami.eks_worker.id + default_ami_id_windows = data.aws_ami.eks_worker_windows.id + workers_group_defaults_defaults = { name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. - ami_id = data.aws_ami.eks_worker.id # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI. + ami_id = "" # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. asg_max_size = "3" # Maximum worker capacity in the autoscaling group. asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. @@ -55,6 +58,7 @@ locals { placement_group = "" # The name of the placement group into which to launch the instances, if any. service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. + platform = "linux" # Platform of workers. either "linux" or "windows" # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device diff --git a/templates/userdata_windows.tpl b/templates/userdata_windows.tpl new file mode 100644 index 0000000000..61be8e8b11 --- /dev/null +++ b/templates/userdata_windows.tpl @@ -0,0 +1,11 @@ + +${pre_userdata} + +[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" +[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' +[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" +& $EKSBootstrapScriptFile -EKSClusterName ${cluster_name} -KubeletExtraArgs '${kubelet_extra_args}' 3>&1 4>&1 5>&1 6>&1 +$LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } + +${additional_userdata} + diff --git a/templates/worker-role.tpl b/templates/worker-role.tpl index 2a9af5863f..f5a00d6d0f 100644 --- a/templates/worker-role.tpl +++ b/templates/worker-role.tpl @@ -3,3 +3,6 @@ groups: - system:bootstrappers - system:nodes + %{~ if platform == "windows" ~} + - eks:kube-proxy-windows + %{~ endif ~} diff --git a/variables.tf b/variables.tf index a4d2d4dc10..9161856318 100644 --- a/variables.tf +++ b/variables.tf @@ -126,12 +126,24 @@ variable "worker_ami_name_filter" { default = "" } +variable "worker_ami_name_filter_windows" { + description = "Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used." + type = string + default = "" +} + variable "worker_ami_owner_id" { description = "The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft')." type = string default = "602401143452" // The ID of the owner of the official AWS EKS AMIs. } +variable "worker_ami_owner_id_windows" { + description = "The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft')." + type = string + default = "801119661308" // The ID of the owner of the official AWS EKS Windows AMIs. +} + variable "worker_additional_security_group_ids" { description = "A list of additional security group ids to attach to worker instances" type = list(string) diff --git a/workers.tf b/workers.tf index ca0bd728d6..a59800d357 100644 --- a/workers.tf +++ b/workers.tf @@ -166,7 +166,7 @@ resource "aws_launch_configuration" "workers" { image_id = lookup( var.worker_groups[count.index], "ami_id", - local.workers_group_defaults["ami_id"], + lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? local.default_ami_id_windows : local.default_ami_id_linux, ) instance_type = lookup( var.worker_groups[count.index], diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 6bb6437b67..fefacd63cb 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -257,7 +257,7 @@ resource "aws_launch_template" "workers_launch_template" { image_id = lookup( var.worker_groups_launch_template[count.index], "ami_id", - local.workers_group_defaults["ami_id"], + lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? local.default_ami_id_windows : local.default_ami_id_linux, ) instance_type = lookup( var.worker_groups_launch_template[count.index], From c46d5fcb00fb7271e0e9f83e299b5828e7a95640 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Mon, 28 Oct 2019 15:08:31 +0100 Subject: [PATCH 087/962] Update PULL_REQUEST_TEMPLATE.md (#573) --- .github/PULL_REQUEST_TEMPLATE.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2d8f78be51..9310a74454 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,7 +6,6 @@ Please explain the changes you made here and link to any relevant issues. ### Checklist -- [ ] `terraform fmt` and `terraform validate` both work from the root and `examples/*` directories +- [ ] Change added to CHANGELOG.md. All changes must be added and breaking changes and highlighted - [ ] CI tests are passing -- [ ] I've added my change to CHANGELOG.md and highlighted any breaking changes - [ ] README.md has been updated after any changes to variables and outputs. See https://github.com/terraform-aws-modules/terraform-aws-eks/#doc-generation From 4f552891ffb6fbed6c17f26ba714e3e99b689a7b Mon Sep 17 00:00:00 2001 From: Scott Stanton Date: Tue, 29 Oct 2019 04:12:47 -0700 Subject: [PATCH 088/962] Allow the userdata template to be replaced (#565) * Allow the userdata template to be replaced * Address fmt issue * Rename 'customer_userdata' name to 'userdata_template_file' * Add support for userdata_template_extra_args * Add support for userdata_template_extra_args * fix merge conflicts * Fix merge problem that dropped the file load Add `platform` to the template expansion context Add `userdata_launch_template` to `launch_template_userdata` --- CHANGELOG.md | 1 + data.tf | 46 ++++++++++++++++++++++++++++++++++------------ local.tf | 2 ++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb343ca936..3b90864d7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - **Breaking:** Allow for specifying a custom AMI for the worker nodes. (by @bmcstdio) - Added support for Windows workers AMIs (by @hodduc) +- Allow for replacing the full userdata text with a `userdata_template_file` template and `userdata_template_extra_args` in `worker_groups` (by @snstanton) - Write your awesome addition here (by @you) ### Changed diff --git a/data.tf b/data.tf index 8fee32f9cf..732efa1c87 100644 --- a/data.tf +++ b/data.tf @@ -105,13 +105,18 @@ EOF data "template_file" "userdata" { count = local.worker_group_count - template = file( - lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? - "${path.module}/templates/userdata_windows.tpl" : - "${path.module}/templates/userdata.sh.tpl" + template = lookup( + var.worker_groups[count.index], + "userdata_template_file", + file( + lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" + ? "${path.module}/templates/userdata_windows.tpl" + : "${path.module}/templates/userdata.sh.tpl" + ) ) - vars = { + vars = merge({ + platform = lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) cluster_name = aws_eks_cluster.this.name endpoint = aws_eks_cluster.this.endpoint cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data @@ -135,18 +140,29 @@ data "template_file" "userdata" { "kubelet_extra_args", local.workers_group_defaults["kubelet_extra_args"], ) - } + }, + lookup( + var.worker_groups[count.index], + "userdata_template_extra_args", + local.workers_group_defaults["userdata_template_extra_args"] + ) + ) } data "template_file" "launch_template_userdata" { count = local.worker_group_launch_template_count - template = file( - lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? - "${path.module}/templates/userdata_windows.tpl" : - "${path.module}/templates/userdata.sh.tpl" + template = lookup( + var.worker_groups_launch_template[count.index], + "userdata_template_file", + file( + lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" + ? "${path.module}/templates/userdata_windows.tpl" + : "${path.module}/templates/userdata.sh.tpl" + ) ) - vars = { + vars = merge({ + platform = lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) cluster_name = aws_eks_cluster.this.name endpoint = aws_eks_cluster.this.endpoint cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data @@ -170,7 +186,13 @@ data "template_file" "launch_template_userdata" { "kubelet_extra_args", local.workers_group_defaults["kubelet_extra_args"], ) - } + }, + lookup( + var.worker_groups_launch_template[count.index], + "userdata_template_extra_args", + local.workers_group_defaults["userdata_template_extra_args"] + ) + ) } data "aws_iam_role" "custom_cluster_iam_role" { diff --git a/local.tf b/local.tf index 7a4b457007..d0c6843d23 100644 --- a/local.tf +++ b/local.tf @@ -40,6 +40,8 @@ locals { root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". key_name = "" # The key name that should be used for the instances in the autoscaling group pre_userdata = "" # userdata to pre-append to the default userdata. + userdata_template_file = "" # alternate template to use for userdata + userdata_template_extra_args = {} # Additional arguments to use when expanding the userdata template file bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI (Amazon Machine Image). additional_userdata = "" # userdata to append to the default userdata. ebs_optimized = true # sets whether to use ebs optimization on supported types. From a8e54ccf73cac2f797f52f878ef1a843c5a8c225 Mon Sep 17 00:00:00 2001 From: Miguel Ferreira Date: Tue, 29 Oct 2019 13:55:21 +0100 Subject: [PATCH 089/962] Use join and splat syntax to access conditional resources (#569) --- CHANGELOG.md | 1 + cluster.tf | 8 ++++---- local.tf | 8 ++++---- workers.tf | 12 ++++++------ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b90864d7a..c26020c7b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - **Breaking:** The `kubectl` configuration file can now be fully-specified using `config_output_path`. Previously it was assumed that `config_output_path` referred to a directory and always ended with a forward slash. This is a breaking change if `config_output_path` does **not** end with a forward slash (which was advised against by the documentation). - Changed logic for setting default ebs_optimized to only require maintaining a list of instance types that don't support it (by @jeffmhastings) - Bumped minimum terraform version to 0.12.2 to prevent an error on yamlencode function (by @toadjaune) +- Access conditional resource using join function in combination with splat syntax (by @miguelaferreira) # History diff --git a/cluster.tf b/cluster.tf index ba33c796e4..7b8a1c9184 100644 --- a/cluster.tf +++ b/cluster.tf @@ -49,7 +49,7 @@ resource "aws_security_group_rule" "cluster_egress_internet" { count = var.cluster_create_security_group ? 1 : 0 description = "Allow cluster egress access to the Internet." protocol = "-1" - security_group_id = aws_security_group.cluster[0].id + security_group_id = local.cluster_security_group_id cidr_blocks = ["0.0.0.0/0"] from_port = 0 to_port = 0 @@ -60,7 +60,7 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { count = var.cluster_create_security_group ? 1 : 0 description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" - security_group_id = aws_security_group.cluster[0].id + security_group_id = local.cluster_security_group_id source_security_group_id = local.worker_security_group_id from_port = 443 to_port = 443 @@ -80,11 +80,11 @@ resource "aws_iam_role" "cluster" { resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" { count = var.manage_cluster_iam_resources ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" - role = aws_iam_role.cluster[0].name + role = local.cluster_iam_role_name } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { count = var.manage_cluster_iam_resources ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" - role = aws_iam_role.cluster[0].name + role = local.cluster_iam_role_name } diff --git a/local.tf b/local.tf index d0c6843d23..d3eccdbad2 100644 --- a/local.tf +++ b/local.tf @@ -8,10 +8,10 @@ locals { ) ] - cluster_security_group_id = var.cluster_create_security_group ? aws_security_group.cluster[0].id : var.cluster_security_group_id - cluster_iam_role_name = var.manage_cluster_iam_resources ? aws_iam_role.cluster[0].name : var.cluster_iam_role_name - cluster_iam_role_arn = var.manage_cluster_iam_resources ? aws_iam_role.cluster[0].arn : data.aws_iam_role.custom_cluster_iam_role[0].arn - worker_security_group_id = var.worker_create_security_group ? aws_security_group.workers[0].id : var.worker_security_group_id + cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id + cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name + cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) + worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name diff --git a/workers.tf b/workers.tf index a59800d357..e946e78e15 100644 --- a/workers.tf +++ b/workers.tf @@ -260,7 +260,7 @@ resource "aws_security_group_rule" "workers_egress_internet" { count = var.worker_create_security_group ? 1 : 0 description = "Allow nodes all egress to the Internet." protocol = "-1" - security_group_id = aws_security_group.workers[0].id + security_group_id = local.worker_security_group_id cidr_blocks = ["0.0.0.0/0"] from_port = 0 to_port = 0 @@ -271,8 +271,8 @@ resource "aws_security_group_rule" "workers_ingress_self" { count = var.worker_create_security_group ? 1 : 0 description = "Allow node to communicate with each other." protocol = "-1" - security_group_id = aws_security_group.workers[0].id - source_security_group_id = aws_security_group.workers[0].id + security_group_id = local.worker_security_group_id + source_security_group_id = local.worker_security_group_id from_port = 0 to_port = 65535 type = "ingress" @@ -282,7 +282,7 @@ resource "aws_security_group_rule" "workers_ingress_cluster" { count = var.worker_create_security_group ? 1 : 0 description = "Allow workers pods to receive communication from the cluster control plane." protocol = "tcp" - security_group_id = aws_security_group.workers[0].id + security_group_id = local.worker_security_group_id source_security_group_id = local.cluster_security_group_id from_port = var.worker_sg_ingress_from_port to_port = 65535 @@ -293,7 +293,7 @@ resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { count = var.worker_create_security_group ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 description = "Allow workers Kubelets to receive communication from the cluster control plane." protocol = "tcp" - security_group_id = aws_security_group.workers[0].id + security_group_id = local.worker_security_group_id source_security_group_id = local.cluster_security_group_id from_port = 10250 to_port = 10250 @@ -304,7 +304,7 @@ resource "aws_security_group_rule" "workers_ingress_cluster_https" { count = var.worker_create_security_group ? 1 : 0 description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane." protocol = "tcp" - security_group_id = aws_security_group.workers[0].id + security_group_id = local.worker_security_group_id source_security_group_id = local.cluster_security_group_id from_port = 443 to_port = 443 From 0d7ef6ffd6ee8647fb5a731de6e1ce08bb0c1168 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 30 Oct 2019 13:38:51 +0100 Subject: [PATCH 090/962] New release 6.1.0 (#576) --- CHANGELOG.md | 15 +++++++++++++-- version | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c26020c7b3..3b06473cb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,19 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v6.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...HEAD)] - 2019-10-??] +## [[v6.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.1.0...HEAD)] - 2019-??-??] + +### Added + +- Write your awesome addition here (by @you) + +### Changed + +- Write your awesome change here (by @you) + +# History + +## [[v6.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v6.1.0)] - 2019-10-30] ### Added @@ -23,7 +35,6 @@ project adheres to [Semantic Versioning](http://semver.org/). - Bumped minimum terraform version to 0.12.2 to prevent an error on yamlencode function (by @toadjaune) - Access conditional resource using join function in combination with splat syntax (by @miguelaferreira) -# History ## [[v6.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2)] - 2019-10-07] diff --git a/version b/version index 860cf3e47f..ed15bef9ae 100644 --- a/version +++ b/version @@ -1 +1 @@ -v6.0.2 +v6.1.0 From 55ff38cc31e8cd965034283a6264c5a8037e3c67 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Wed, 30 Oct 2019 21:22:15 +0100 Subject: [PATCH 091/962] Fix changelog for release v7.0.0 (#577) * New v7.0.0 release * update version file * remove one line --- CHANGELOG.md | 6 ++---- version | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b06473cb6..b3901b9cb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v6.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.1.0...HEAD)] - 2019-??-??] +## [[v7.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.1.0...HEAD)] - 2019-??-??] ### Added @@ -19,14 +19,13 @@ project adheres to [Semantic Versioning](http://semver.org/). # History -## [[v6.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v6.1.0)] - 2019-10-30] +## [[v7.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v7.0.0)] - 2019-10-30] ### Added - **Breaking:** Allow for specifying a custom AMI for the worker nodes. (by @bmcstdio) - Added support for Windows workers AMIs (by @hodduc) - Allow for replacing the full userdata text with a `userdata_template_file` template and `userdata_template_extra_args` in `worker_groups` (by @snstanton) -- Write your awesome addition here (by @you) ### Changed @@ -35,7 +34,6 @@ project adheres to [Semantic Versioning](http://semver.org/). - Bumped minimum terraform version to 0.12.2 to prevent an error on yamlencode function (by @toadjaune) - Access conditional resource using join function in combination with splat syntax (by @miguelaferreira) - ## [[v6.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2)] - 2019-10-07] ### Added diff --git a/version b/version index ed15bef9ae..d18077c214 100644 --- a/version +++ b/version @@ -1 +1 @@ -v6.1.0 +v7.0.0 From c0ae644204f237d8f7c6e97b1a5789ca9f418ed7 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Mon, 4 Nov 2019 16:18:19 +0100 Subject: [PATCH 092/962] Add Action check against minimum versions (#575) --- .github/workflows/lint.yml | 20 ++++++++++++++++++-- CHANGELOG.md | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 53cac00ad1..a9eedc8aa9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@master - name: TFLint uses: docker://wata727/tflint - + fmt: name: Code Format runs-on: ubuntu-latest @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@master - run: terraform fmt --recursive -check=true - + docs: name: Docs runs-on: macOS-latest @@ -51,3 +51,19 @@ jobs: terraform validate cd - done + + minimum: + name: Minimum version check + runs-on: ubuntu-latest + container: + image: hashicorp/terraform:0.12.2 + steps: + - uses: actions/checkout@master + - name: Validate Code + env: + AWS_REGION: 'us-east-1' + TF_WARN_OUTPUT_ERRORS: 1 + run: | + sed -i -e 's/>=/=/' -e 's/ \(\d\+\.\d\+\)"/ \1.0"/' versions.tf + terraform init + terraform validate -var "region=${AWS_REGION}" -var "vpc_id=vpc-123456" -var "subnets=[\"subnet-12345a\"]" -var "workers_ami_id=ami-123456" -var "cluster_ingress_cidrs=[]" -var "cluster_name=test_cluster" diff --git a/CHANGELOG.md b/CHANGELOG.md index b3901b9cb1..d025b06c03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added -- Write your awesome addition here (by @you) +- Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) ### Changed From a43e44f16490fa904922d405bd5a303dc84fc104 Mon Sep 17 00:00:00 2001 From: Ankit Wal Date: Thu, 7 Nov 2019 00:27:56 +0800 Subject: [PATCH 093/962] Update worker_iam_instance_profile outputs for launch template use (#583) * Add launch template instance profiles to output Concat launch template and asg instance profile lists Signed-off-by: Ankit Wal * Update ChangeLog Added changes made to output in Changelog Signed-off-by: Ankit Wal --- CHANGELOG.md | 2 +- outputs.tf | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d025b06c03..057ff39dc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed -- Write your awesome change here (by @you) +- Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) # History diff --git a/outputs.tf b/outputs.tf index a195b21fcf..d8755039ce 100644 --- a/outputs.tf +++ b/outputs.tf @@ -114,12 +114,18 @@ output "worker_security_group_id" { output "worker_iam_instance_profile_arns" { description = "default IAM instance profile ARN for EKS worker groups" - value = aws_iam_instance_profile.workers.*.arn + value = concat( + aws_iam_instance_profile.workers.*.arn, + aws_iam_instance_profile.workers_launch_template.*.arn + ) } output "worker_iam_instance_profile_names" { description = "default IAM instance profile name for EKS worker groups" - value = aws_iam_instance_profile.workers.*.name + value = concat( + aws_iam_instance_profile.workers.*.name, + aws_iam_instance_profile.workers_launch_template.*.name + ) } output "worker_iam_role_name" { From a4117177f6a14b3d5c278eb16d1e3d38fba9932b Mon Sep 17 00:00:00 2001 From: Hwanjin Jeong Date: Tue, 19 Nov 2019 02:10:53 +0900 Subject: [PATCH 094/962] Fix typo (#595) * fix typos * remove unnecessary comment --- README.md | 4 ++-- data.tf | 1 - variables.tf | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index edcc733ced..7a7063a21f 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | string | `""` | no | | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | -| cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers | string | `""` | no | +| cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | string | `""` | no | | cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.14"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | string | `"./"` | no | | iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | @@ -150,7 +150,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | -| worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster. | string | `""` | no | +| worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | string | `""` | no | | worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | number | `"1025"` | no | | workers\_additional\_policies | Additional policies to be added to workers | list(string) | `[]` | no | | workers\_group\_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | any | `{}` | no | diff --git a/data.tf b/data.tf index 732efa1c87..e66acc3955 100644 --- a/data.tf +++ b/data.tf @@ -42,7 +42,6 @@ data "aws_ami" "eks_worker_windows" { most_recent = true - # Owner ID of AWS EKS team (windows) owners = [var.worker_ami_owner_id_windows] } diff --git a/variables.tf b/variables.tf index 9161856318..06c9460fb8 100644 --- a/variables.tf +++ b/variables.tf @@ -20,7 +20,7 @@ variable "cluster_name" { } variable "cluster_security_group_id" { - description = "If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers" + description = "If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers" type = string default = "" } @@ -115,7 +115,7 @@ variable "worker_groups_launch_template" { } variable "worker_security_group_id" { - description = "If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster." + description = "If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster." type = string default = "" } From 4f3f5b6733d3ca16206883948e84e160ed970e58 Mon Sep 17 00:00:00 2001 From: Angel Abad Date: Mon, 18 Nov 2019 18:16:37 +0100 Subject: [PATCH 095/962] Fix deprecated interpolation-only expression (#594) --- CHANGELOG.md | 1 + kubectl.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 057ff39dc8..32c9f6a3a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed - Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) +- Fix deprecated interpolation-only expression (by @angelabad) # History diff --git a/kubectl.tf b/kubectl.tf index 5a70828f0d..c7b86d4698 100644 --- a/kubectl.tf +++ b/kubectl.tf @@ -1,6 +1,6 @@ resource "local_file" "kubeconfig" { count = var.write_kubeconfig ? 1 : 0 content = data.template_file.kubeconfig.rendered - filename = "${substr(var.config_output_path, -1, 1) == "/" ? "${var.config_output_path}kubeconfig_${var.cluster_name}" : var.config_output_path}" + filename = substr(var.config_output_path, -1, 1) == "/" ? "${var.config_output_path}kubeconfig_${var.cluster_name}" : var.config_output_path } From 215dc849e8775a0b3366f02dbf203797f39e7596 Mon Sep 17 00:00:00 2001 From: Joonsung Lee Date: Wed, 20 Nov 2019 21:19:06 +0900 Subject: [PATCH 096/962] Fix broken terraform plan/apply on a cluster < 1.14 (Fix #582) (#600) --- CHANGELOG.md | 1 + data.tf | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32c9f6a3a3..c1f045b366 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) - Fix deprecated interpolation-only expression (by @angelabad) +- Fix broken terraform plan/apply on a cluster < 1.14 (by @hodduc) # History diff --git a/data.tf b/data.tf index e66acc3955..20fd35674b 100644 --- a/data.tf +++ b/data.tf @@ -1,6 +1,10 @@ locals { - worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" - worker_ami_name_filter_windows = var.worker_ami_name_filter_windows != "" ? var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${var.cluster_version}-*" + worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" + + # Windows nodes are available from k8s 1.14. If cluster version is less than 1.14, fix ami filter to some constant to not fail on 'terraform plan'. + worker_ami_name_filter_windows = (var.worker_ami_name_filter_windows != "" ? + var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${tonumber(var.cluster_version) >= 1.14 ? var.cluster_version : 1.14}-*" + ) } data "aws_iam_policy_document" "workers_assume_role_policy" { From b69c8fb7596de59aa69fee020f8941ec6618f6b4 Mon Sep 17 00:00:00 2001 From: Mary Hayes Date: Wed, 20 Nov 2019 06:23:56 -0600 Subject: [PATCH 097/962] Run aws-auth Update Commands in Sequence and not Parallel (#592) * Run aws-auth Update Commands in Sequence and not Parallel * Update changelog --- CHANGELOG.md | 2 ++ aws_auth.tf | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1f045b366..35a7a40dbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed - Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) +- Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) +- Exit with error code when `aws-auth` configmap is unable to be updated (by @knittingdev) - Fix deprecated interpolation-only expression (by @angelabad) - Fix broken terraform plan/apply on a cluster < 1.14 (by @hodduc) diff --git a/aws_auth.tf b/aws_auth.tf index bd3221194d..ebec0351be 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -12,13 +12,16 @@ resource "null_resource" "update_config_map_aws_auth" { working_dir = path.module command = < kube_config.yaml & \ -echo "${null_resource.update_config_map_aws_auth[0].triggers.config_map_rendered}" > aws_auth_configmap.yaml & \ -kubectl apply -f aws_auth_configmap.yaml --kubeconfig kube_config.yaml && break || \ +echo "${null_resource.update_config_map_aws_auth[0].triggers.kube_config_map_rendered}" > kube_config.yaml && \ +echo "${null_resource.update_config_map_aws_auth[0].triggers.config_map_rendered}" > aws_auth_configmap.yaml && \ +kubectl apply -f aws_auth_configmap.yaml --kubeconfig kube_config.yaml && \ +completed_apply=1 && break || \ sleep 10; \ done; \ rm aws_auth_configmap.yaml kube_config.yaml; +if [ "$completed_apply" = "0" ]; then exit 1; fi; EOS From 93636625740c63fd89ad8bc60ad180761288c54d Mon Sep 17 00:00:00 2001 From: Stijn De Haes Date: Thu, 28 Nov 2019 10:25:13 +0100 Subject: [PATCH 098/962] Use kubernetes provider to manage aws auth (#355) This commit changes the way aws auth is managed. Before a local file was used the generate the template and a null resource to apply it. This is now switched to the terraform kubernetes provider. --- CHANGELOG.md | 3 +- README.md | 22 ++++++- aws_auth.tf | 66 ++++--------------- ...grading-to-aws-auth-kubernetes-provider.md | 14 ++++ examples/basic/main.tf | 16 +++++ examples/launch_templates/main.tf | 16 +++++ examples/spot_instances/main.tf | 16 +++++ outputs.tf | 2 +- templates/config-map-aws-auth.yaml.tpl | 19 ------ templates/worker-role.tpl | 16 ++--- 10 files changed, 108 insertions(+), 82 deletions(-) create mode 100644 docs/upgrading-to-aws-auth-kubernetes-provider.md delete mode 100644 templates/config-map-aws-auth.yaml.tpl diff --git a/CHANGELOG.md b/CHANGELOG.md index 35a7a40dbc..b5e8df4eb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v7.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.1.0...HEAD)] - 2019-??-??] +## [[v7.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...HEAD)] - 2019-??-??] ### Added @@ -16,6 +16,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Changed - Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) +- **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. Read the [docs](docs/upgrading-to-aws-auth-kubernetes-provider.md) for more info (by @sdehaes) - Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) - Exit with error code when `aws-auth` configmap is unable to be updated (by @knittingdev) - Fix deprecated interpolation-only expression (by @angelabad) diff --git a/README.md b/README.md index 7a7063a21f..1ff10e5eb7 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,29 @@ Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aw ## Usage example -A full example leveraging other community modules is contained in the [examples/basic directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). Here's the gist of using it via the Terraform registry: +A full example leveraging other community modules is contained in the [examples/basic directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). +Please do not forget to set the provider to the EKS cluster. This is needed to provision the aws_auth configmap in +kube-system. You can also use this provider to create your own kubernetes resources with the terraform kubernetes +provider. +Here's the gist of using it via the Terraform registry: ```hcl +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false + version = "~> 1.9" +} + module "my-cluster" { source = "terraform-aws-modules/eks/aws" cluster_name = "my-cluster" diff --git a/aws_auth.tf b/aws_auth.tf index ebec0351be..263ceab6e8 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -1,40 +1,3 @@ -resource "local_file" "config_map_aws_auth" { - count = var.write_aws_auth_config ? 1 : 0 - content = data.template_file.config_map_aws_auth.rendered - filename = "${var.config_output_path}config-map-aws-auth_${var.cluster_name}.yaml" -} - -resource "null_resource" "update_config_map_aws_auth" { - count = var.manage_aws_auth ? 1 : 0 - depends_on = [aws_eks_cluster.this] - - provisioner "local-exec" { - working_dir = path.module - - command = < kube_config.yaml && \ -echo "${null_resource.update_config_map_aws_auth[0].triggers.config_map_rendered}" > aws_auth_configmap.yaml && \ -kubectl apply -f aws_auth_configmap.yaml --kubeconfig kube_config.yaml && \ -completed_apply=1 && break || \ -sleep 10; \ -done; \ -rm aws_auth_configmap.yaml kube_config.yaml; -if [ "$completed_apply" = "0" ]; then exit 1; fi; -EOS - - - interpreter = var.local_exec_interpreter - } - - triggers = { - kube_config_map_rendered = data.template_file.kubeconfig.rendered - config_map_rendered = data.template_file.config_map_aws_auth.rendered - endpoint = aws_eks_cluster.this.endpoint - } -} - data "aws_caller_identity" "current" { } @@ -79,21 +42,20 @@ data "template_file" "worker_role_arns" { } } -data "template_file" "config_map_aws_auth" { - template = file("${path.module}/templates/config-map-aws-auth.yaml.tpl") +resource "kubernetes_config_map" "aws_auth" { + count = var.manage_aws_auth ? 1 : 0 - vars = { - worker_role_arn = join( - "", - distinct( - concat( - data.template_file.launch_template_worker_role_arns.*.rendered, - data.template_file.worker_role_arns.*.rendered, - ), - ), - ) - map_users = yamlencode(var.map_users), - map_roles = yamlencode(var.map_roles), - map_accounts = yamlencode(var.map_accounts) + metadata { + name = "aws-auth" + namespace = "kube-system" + } + + data = { + mapRoles = < Date: Thu, 28 Nov 2019 15:03:26 +0100 Subject: [PATCH 099/962] Misc doc and changelog format changes (#604) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 👊 --- CHANGELOG.md | 215 +++++------------- README.md | 28 +-- docs/spot-instances.md | 6 +- ...grading-to-aws-auth-kubernetes-provider.md | 14 -- 4 files changed, 68 insertions(+), 195 deletions(-) delete mode 100644 docs/upgrading-to-aws-auth-kubernetes-provider.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b5e8df4eb6..abee4af9cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v7.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...HEAD)] - 2019-??-??] -### Added - - Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) - -### Changed - - Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) - **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. Read the [docs](docs/upgrading-to-aws-auth-kubernetes-provider.md) for more info (by @sdehaes) - Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) @@ -22,104 +17,97 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fix deprecated interpolation-only expression (by @angelabad) - Fix broken terraform plan/apply on a cluster < 1.14 (by @hodduc) +#### Important notes + +The way the `aws-auth` configmap in the `kube-system` namespaces is managed has been changed. Before this was managed via kubectl using a null resources. This was changed to be managed by the terraform Kubernetes provider. + +To upgrade you have to add the kubernetes provider to the place you are calling the module. You can see examples in +the [examples](../examples) folder. Then you should import the configmap into Terraform: + +``` +terraform import module.cluster1.kubernetes_config_map.aws_auth[0] kube-system/aws-auth +``` + +You could also delete the aws-auth config map before doing an apply but this means you need to the apply with the **same user/role that created the cluster**. + # History ## [[v7.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v7.0.0)] - 2019-10-30] -### Added - - **Breaking:** Allow for specifying a custom AMI for the worker nodes. (by @bmcstdio) - Added support for Windows workers AMIs (by @hodduc) - Allow for replacing the full userdata text with a `userdata_template_file` template and `userdata_template_extra_args` in `worker_groups` (by @snstanton) - -### Changed - - **Breaking:** The `kubectl` configuration file can now be fully-specified using `config_output_path`. Previously it was assumed that `config_output_path` referred to a directory and always ended with a forward slash. This is a breaking change if `config_output_path` does **not** end with a forward slash (which was advised against by the documentation). - Changed logic for setting default ebs_optimized to only require maintaining a list of instance types that don't support it (by @jeffmhastings) - Bumped minimum terraform version to 0.12.2 to prevent an error on yamlencode function (by @toadjaune) - Access conditional resource using join function in combination with splat syntax (by @miguelaferreira) -## [[v6.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2)] - 2019-10-07] +#### Important notes -### Added +An AMI is now specified using the whole name, for example `amazon-eks-node-1.14-v20190927`. -- Added `tags` to `aws_eks_cluster` introduced by terraform-provider-aws 2.31.0 (by @morganchristiansson) - -### Changed +## [[v6.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2)] - 2019-10-07] - - Add option to enable lifecycle hooks creation (by @barryib) - - Remove helm chart value `sslCertPath` described in `docs/autoscaling.md` (by @wi1dcard) - - Attaching of IAM policies for autoscaler and CNI to the worker nodes now optional (by @dpiddockcmp) +- Added `tags` to `aws_eks_cluster` introduced by terraform-provider-aws 2.31.0 (by @morganchristiansson) +- Add option to enable lifecycle hooks creation (by @barryib) +- Remove helm chart value `sslCertPath` described in `docs/autoscaling.md` (by @wi1dcard) +- Attaching of IAM policies for autoscaler and CNI to the worker nodes now optional (by @dpiddockcmp) ## [[v6.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...v6.0.1)] - 2019-09-25] -### Added - - - Added support for different workers AMI's, i.e. with GPU support (by @rvoitenko) - -### Changed - +- Added support for different workers AMI's, i.e. with GPU support (by @rvoitenko) - Use null as default value for `target_group_arns` attribute of worker autoscaling group (by @tatusl) - Output empty string when cluster identity is empty (by @tbarry) ## [[v6.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v6.0.0)] - 2019-09-17] - - Added `market_type` to `workers_launch_template.tf` allow the usage of spot nodegroups without mixed instances policy. - - Added support for log group tag in `./cluster.tf` (@lucas-giaco) - - Added support for workers iam role tag in `./workers.tf` (@lucas-giaco) - - Added `required_providers` to enforce provider minimum versions (by @dpiddockcmp) - - Updated `local.spot_allocation_strategy` docstring to indicate availability of new `capacity-optimized` option. (by @sc250024) - - Added support for initial lifecycle hooks for autosacling groups (@barryib) - - Added option to recreate ASG when LT or LC changes (by @barryib) - - Ability to specify workers role name (by @ivanich) - - Added output for OIDC Issuer URL (by @russwhelan) - - Added support for Mixed Instance ASG using `worker_groups_launch_template` variable (by @sppwf) - - Changed ASG Tags generation using terraform 12 `for` utility (by @sppwf) - - **Breaking:** Removed `worker_groups_launch_template_mixed` variable (by @sppwf) - -### Changed - - - Update to EKS 1.14 (by @nauxliu) - - **Breaking:** Support map users and roles to multiple groups (by @nauxliu) - - Fixed errors sometimes happening during destroy due to usage of coalesce() in local.tf (by @petrikero) - - Removed historical mention of adding caller's IPv4 to cluster security group (by @dpiddockcmp) - - Wrapped `kubelet_extra_args` in double quotes instead of singe quotes (by @nxf5025) - - Make terraform plan more consistent and avoid unnecessary "(known after apply)" (by @barryib) - - Made sure that `market_type` was correctly passed to `workers_launch_template` (by @to266) +- Added `market_type` to `workers_launch_template.tf` allow the usage of spot nodegroups without mixed instances policy. +- Added support for log group tag in `./cluster.tf` (@lucas-giaco) +- Added support for workers iam role tag in `./workers.tf` (@lucas-giaco) +- Added `required_providers` to enforce provider minimum versions (by @dpiddockcmp) +- Updated `local.spot_allocation_strategy` docstring to indicate availability of new `capacity-optimized` option. (by @sc250024) +- Added support for initial lifecycle hooks for autosacling groups (@barryib) +- Added option to recreate ASG when LT or LC changes (by @barryib) +- Ability to specify workers role name (by @ivanich) +- Added output for OIDC Issuer URL (by @russwhelan) +- Added support for Mixed Instance ASG using `worker_groups_launch_template` variable (by @sppwf) +- Changed ASG Tags generation using terraform 12 `for` utility (by @sppwf) +- **Breaking:** Removed `worker_groups_launch_template_mixed` variable (by @sppwf) +- Update to EKS 1.14 (by @nauxliu) +- **Breaking:** Support map users and roles to multiple groups (by @nauxliu) +- Fixed errors sometimes happening during destroy due to usage of coalesce() in local.tf (by @petrikero) +- Removed historical mention of adding caller's IPv4 to cluster security group (by @dpiddockcmp) +- Wrapped `kubelet_extra_args` in double quotes instead of singe quotes (by @nxf5025) +- Make terraform plan more consistent and avoid unnecessary "(known after apply)" (by @barryib) +- Made sure that `market_type` was correctly passed to `workers_launch_template` (by @to266) + +#### Important notes + +You will need to move worker groups from `worker_groups_launch_template_mixed` to `worker_groups_launch_template`. You can rename terraform resources in the state to avoid an destructive changes. + +Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [""]`. ## [[v5.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v5.1.1)] - 2019-07-30] -### Added - - - Added new tag in `worker.tf` with autoscaling_enabled = true flag (by @insider89) +- Added new tag in `worker.tf` with autoscaling_enabled = true flag (by @insider89) ## [[v5.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...v5.1.0)] - 2019-07-30] -### Added - - - Option to set a KMS key for the log group and encrypt it (by @till-krauss) - - Output the name of the cloudwatch log group (by @gbooth27) - - Added `cpu_credits` param for the workers defined in `worker_groups_launch_template` (by @a-shink) - - Added support for EBS Volumes tag in `worker_groups_launch_template` and `workers_launch_template_mixed.tf` (by @sppwf) - - Basic example now tags networks correctly, as per [ELB documentation](https://docs.aws.amazon.com/eks/latest/userguide/load-balancing.html) and [ALB documentation](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) (by @karolinepauls) - -### Changed - - - Update default override instance types to work with Cluster Autoscaler (by @nauxliu on behalf of RightCapital) - - Examples now specify `enable_dns_hostnames = true`, as per [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html) (by @karolinepauls) +- Option to set a KMS key for the log group and encrypt it (by @till-krauss) +- Output the name of the cloudwatch log group (by @gbooth27) +- Added `cpu_credits` param for the workers defined in `worker_groups_launch_template` (by @a-shink) +- Added support for EBS Volumes tag in `worker_groups_launch_template` and `workers_launch_template_mixed.tf` (by @sppwf) +- Basic example now tags networks correctly, as per [ELB documentation](https://docs.aws.amazon.com/eks/latest/userguide/load-balancing.html) and [ALB documentation](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) (by @karolinepauls) +- Update default override instance types to work with Cluster Autoscaler (by @nauxliu on behalf of RightCapital) +- Examples now specify `enable_dns_hostnames = true`, as per [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html) (by @karolinepauls) ## [[v5.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...v5.0.0)] - 2019-06-19] -### Added - - Added Termination Policy Option to worker ASGs (by @undeadops) - Update EBS optimized instances type (by @gloutsch) - Added tagging for iam role created in `./cluster.tf` (@camilosantana) - Enable log retention for cloudwatch log groups (by @yuriipolishchuk) - Update to EKS 1.13 (by @gloutsch) - -### Changed - - Finally, Terraform 0.12 support, [Upgrade Guide](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/394) (by @alex-goncharov @nauxliu @timboven) - All the xx_count variables have been removed (by @nauxliu on behalf of RightCapital) - Use actual lists in the workers group maps instead of strings with commas (by @nauxliu on behalf of RightCapital) @@ -128,9 +116,6 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fix toggle for IAM instance profile creation for mixed launch templates (by @jnozo) ## [[v4.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.1...v4.0.2)] - 2019-05-07] - -### Changed - - Added 2 new examples, also tidy up basic example (by @max-rocket-internet) - Updates to travis, PR template (by @max-rocket-internet) - Fix typo in data.tf (by @max-rocket-internet) @@ -138,23 +123,16 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v4.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1)] - 2019-05-07] -### Changed - - Fix annoying typo: worker_group_xx vs worker_groups_xx (by @max-rocket-internet) ## [[v4.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v3.0.0...v4.0.0)] - 2019-05-07] -### Added - - Added support for custom service linked role for Auto Scaling group (by @voanhduy1512) - Added support for custom IAM roles for cluster and workers (by @erks) - Added cluster ARN to outputs (by @alexsn) - Added outputs for `workers_user_data` and `workers_default_ami_id` (by @max-rocket-internet) - Added doc about spot instances (by @max-rocket-internet) - Added new worker group option with a mixed instances policy (by @max-rocket-internet) - -### Changed - - Set default suspended processes for ASG to `AZRebalance` (by @max-rocket-internet) - 4 small changes to `aws_launch_template` resource (by @max-rocket-internet) - (Breaking Change) Rewritten and de-duplicated code related to Launch Templates (by @max-rocket-internet) @@ -164,32 +142,20 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v3.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.1...v3.0.0)] - 2019-04-15] -### Added - - Fixed: Ability to destroy clusters due to security groups being attached to ENI's (by @whiskeyjimbo) - Added outputs for worker IAM instance profile(s) (by @soapergem) - Added support for cluster logging via the `cluster_enabled_log_types` variable (by @sc250024) - -### Changed - - - Updated vpc module version and aws provider version. (by @chenrui333) - - Upgraded default kubernetes version from 1.11 to 1.12 (by @stijndehaes) +- Updated vpc module version and aws provider version. (by @chenrui333) +- Upgraded default kubernetes version from 1.11 to 1.12 (by @stijndehaes) ## [[v2.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.0...v2.3.1)] - 2019-03-26] -### Added - - Added support for eks public and private endpoints (by @stijndehaes) - Added minimum inbound traffic rule to the cluster worker security group as per the [EKS security group requirements](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) (by @sc250024) - -### Changed - - (Breaking Change) Replaced `enable_docker_bridge` with a generic option called `bootstrap_extra_args` to resolve [310](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/310) (by @max-rocket-internet) ## [[v2.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.3.0)] - 2019-03-20] -### Added - - Allow additional policies to be attached to worker nodes (by @rottenbytes) - Ability to specify a placement group for each worker group (by @matheuss) - "k8s.io/cluster-autoscaler/{cluster-name}" and "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" tags for autoscaling groups (by @tbarrella) @@ -202,37 +168,25 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v2.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.2.2)] - 2019-02-25] -### Added - - Ability to specify a path for IAM roles (by @tekn0ir) ## [[v2.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.0...v2.2.1)] - 2019-02-18] ## [[v2.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.1.0...v2.2.0)] - 2019-02-07] -### Added - - Ability to specify a permissions_boundary for IAM roles (by @dylanhellems) - Ability to configure force_delete for the worker group ASG (by @stefansedich) - Ability to configure worker group ASG tags (by @stefansedich) - Added EBS optimized mapping for the g3s.xlarge instance type (by @stefansedich) - `enabled_metrics` input (by @zanitete) - write_aws_auth_config to input (by @yutachaos) - -### Changed - - Change worker group ASG to use create_before_destroy (by @stefansedich) - Fixed a bug where worker group defaults were being used for launch template user data (by @leonsodhi-lf) - Managed_aws_auth option is true, the aws-auth configmap file is no longer created, and write_aws_auth_config must be set to true to generate config_map. (by @yutachaos) ## [[v2.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.0.0...v2.1.0)] - 2019-01-15] -### Added - - Initial support for worker groups based on Launch Templates (by @skang0601) - -### Changed - - Updated the `update_config_map_aws_auth` resource to trigger when the EKS cluster endpoint changes. This likely means that a new cluster was spun up so our ConfigMap won't exist (fixes #234) (by @elatt) - Removed invalid action from worker_autoscaling iam policy (by @marcelloromani) - Fixed zsh-specific syntax in retry loop for aws auth config map (by @marcelloromani) @@ -240,14 +194,9 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v2.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.8.0...v2.0.0)] - 2018-12-14] -### Added - - (Breaking Change) New input variables `map_accounts_count`, `map_roles_count` and `map_users_count` to allow using computed values as part of `map_accounts`, `map_roles` and `map_users` configs (by @chili-man on behalf of OpenGov). - (Breaking Change) New variables `cluster_create_security_group` and `worker_create_security_group` to stop `value of 'count' cannot be computed` error. - Added ability to choose local-exec interpreter (by @rothandrew) - -### Changed - - Added `--with-aggregate-type-defaults` option to terraform-docs (by @max-rocket-internet) - Updated AMI ID filtering to only filter AMIs from current cluster k8s version (by @max-rocket-internet) - Added `pre-commit-terraform` git hook to automatically create documentation of inputs/outputs (by @antonbabenko) @@ -256,17 +205,12 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v1.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.7.0...v1.8.0)] - 2018-12-04] -### Added - - Support for using AWS Launch Templates to define autoscaling groups (by @skang0601) - `suspended_processes` to `worker_groups` input (by @bkmeneguello) - `target_group_arns` to `worker_groups` input (by @zihaoyu) - `force_detach_policies` to `aws_iam_role` `cluster` and `workers` (by @marky-mark) - Added sleep while trying to apply the kubernetes configurations if failed, up to 50 seconds (by @rmakram-ims) - `cluster_create_security_group` and `worker_create_security_group`. This allows using computed cluster and worker security groups. (by @rmakram-ims) - -### Changed - - new variables worker_groups_launch_template and worker_group_count_launch_template (by @skang0601) - Remove aws_iam_service_linked_role (by @max-rocket-internet) - Adjust the order and correct/update the ec2 instance type info. (@chenrui333) @@ -275,15 +219,10 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v1.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.6.0...v1.7.0)] - 2018-10-09] -### Added - - Worker groups can be created with a specified IAM profile. (from @laverya) - exposed `aws_eks_cluster` create and destroy timeouts (by @RGPosadas) - exposed `placement_tenancy` for autoscaling group (by @monsterxx03) - Allow port 443 from EKS service to nodes to run `metrics-server`. (by @max-rocket-internet) - -### Changed - - fix default worker subnets not working (by @erks) - fix default worker autoscaling_enabled not working (by @erks) - Cosmetic syntax changes to improve readability. (by @max-rocket-internet) @@ -291,15 +230,10 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v1.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.5.0...v1.6.0)] - 2018-09-04] -### Added - - add support for [`amazon-eks-node-*` AMI with bootstrap script](https://aws.amazon.com/blogs/opensource/improvements-eks-worker-node-provisioning/) (by @erks) - expose `kubelet_extra_args` worker group option (replacing `kubelet_node_labels`) to allow specifying arbitrary kubelet options (e.g. taints and labels) (by @erks) - add optional input `worker_additional_security_group_ids` to allow one or more additional security groups to be added to all worker launch configurations - #47 (by @hhobbsh @mr-joshua) - add optional input `additional_security_group_ids` to allow one or more additional security groups to be added to a specific worker launch configuration - #47 (by @mr-joshua) - -### Changed - - allow a custom AMI to be specified as a default (by @erks) - bugfix for above change (by @max-rocket-internet) - **Breaking change** Removed support for `eks-worker-*` AMI. The cluster specifying a custom AMI based off of `eks-worker-*` AMI will have to rebuild the AMI from `amazon-eks-node-*`. (by @erks) @@ -307,48 +241,31 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v1.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.4.0...v1.5.0)] - 2018-08-30] -### Added - - add spot_price option to aws_launch_configuration - add enable_monitoring option to aws_launch_configuration - add t3 instance class settings - add aws_iam_service_linked_role for elasticloadbalancing. (by @max-rocket-internet) - Added autoscaling policies into module that are optionally attached when enabled for a worker group. (by @max-rocket-internet) - -### Changed - - **Breaking change** Removed `workstation_cidr` variable, http callout and unnecessary security rule. (by @dpiddockcmp) If you are upgrading from 1.4 you should fix state after upgrade: `terraform state rm module.eks.data.http.workstation_external_ip` - Can now selectively override keys in `workers_group_defaults` variable rather than callers maintaining a duplicate of the whole map. (by @dpiddockcmp) ## [[v1.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.3.0...v1.4.0)] - 2018-08-02] -### Added - - manage eks workers' root volume size and type. - `workers_asg_names` added to outputs. (kudos to @laverya) - New top level variable `worker_group_count` added to replace the use of `length(var.worker_groups)`. This allows using computed values as part of worker group configs. (complaints to @laverya) ## [[v1.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.2.0...v1.3.0)] - 2018-07-11] -### Added - - New variables `map_accounts`, `map_roles` and `map_users` in order to manage additional entries in the `aws-auth` configmap. (by @max-rocket-internet) - kubelet_node_labels worker group option allows setting --node-labels= in kubelet. (Hat-tip, @bshelton229 👒) - `worker_iam_role_arn` added to outputs. Sweet, @hatemosphere 🔥 - -### Changed - - Worker subnets able to be specified as a dedicated list per autoscaling group. (up top, @bshelton229 🙏) ## [[v1.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.1.0...v1.2.0)] - 2018-07-01] -### Added - - new variable `pre_userdata` added to worker launch configuration allows to run scripts before the plugin does anything. (W00t, @jimbeck 🦉) - -### Changed - - kubeconfig made much more flexible. (Bang up job, @sdavids13 💥) - ASG desired capacity is now ignored as ASG size is more effectively handed by k8s. (Thanks, @ozbillwang 💇‍♂️) - Providing security groups didn't behave as expected. This has been fixed. (Good catch, @jimbeck 🔧) @@ -356,54 +273,34 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v1.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.0.0...v1.1.0)] - 2018-06-25] -### Added - - new variable `worker_sg_ingress_from_port` allows to change the minimum port number from which pods will accept communication (Thanks, @ilyasotkov 👏). - expanded on worker example to show how multiple worker autoscaling groups can be created. - IPv4 is used explicitly to resolve testing from IPv6 networks (thanks, @tsub 🙏). - Configurable public IP attachment and ssh keys for worker groups. Defaults defined in `worker_group_defaults`. Nice, @hatemosphere 🌂 - `worker_iam_role_name` now an output. Sweet, @artursmet 🕶️ - -### Changed - - IAM test role repaired by @lcharkiewicz 💅 - `kube-proxy` restart no longer needed in userdata. Good catch, @hatemosphere 🔥 - worker ASG reattachment wasn't possible when using `name`. Moved to `name_prefix` to allow recreation of resources. Kudos again, @hatemosphere 🐧 ## [[v1.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.2.0...v1.0.0)] - 2018-06-11] -### Added - - security group id can be provided for either/both of the cluster and the workers. If not provided, security groups will be created with sufficient rules to allow cluster-worker communication. - kudos to @tanmng on the idea ⭐ - outputs of security group ids and worker ASG arns added for working with these resources outside the module. - -### Changed - - Worker build out refactored to allow multiple autoscaling groups each having differing specs. If none are given, a single ASG is created with a set of sane defaults - big thanks to @kppullin 🥨 ## [[v0.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.1...v0.2.0)] - 2018-06-08] -### Added - - ability to specify extra userdata code to execute following kubelet services start. - EBS optimization used whenever possible for the given instance type. - When `configure_kubectl_session` is set to true the current shell will be configured to talk to the kubernetes cluster using config files output from the module. - -### Changed - - files rendered from dedicated templates to separate out raw code and config from `hcl` - `workers_ami_id` is now made optional. If not specified, the module will source the latest AWS supported EKS AMI instead. ## [[v0.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.0...v0.1.1)] - 2018-06-07] - -### Changed - - Pre-commit hooks fixed and working. - Made progress on CI, advancing the build to the final `kitchen test` stage before failing. ## [v0.1.0] - 2018-06-07 -### Added - - Everything! Initial release of the module. - added a local variable to do a lookup against for a dynamic value in userdata which was previously static. Kudos to @tanmng for finding and fixing bug #1! diff --git a/README.md b/README.md index 1ff10e5eb7..6e534eae84 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -# terraform-aws-eks +# terraform-aws-eks [![Lint Status](https://github.com/terraform-aws-modules/terraform-aws-eks/workflows/Lint/badge.svg)](https://github.com/terraform-aws-modules/terraform-aws-eks/actions) -[![LICENSE](https://img.shields.io/github/license/terraform-aws-modules/terraform-aws-eks)](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/LICENSE) +[![LICENSE](https://img.shields.io/github/license/terraform-aws-modules/terraform-aws-eks)](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/LICENSE) A terraform module to create a managed Kubernetes cluster on AWS EKS. Available through the [Terraform registry](https://registry.terraform.io/modules/terraform-aws-modules/eks/aws). @@ -19,10 +19,6 @@ Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aw ## Usage example A full example leveraging other community modules is contained in the [examples/basic directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). -Please do not forget to set the provider to the EKS cluster. This is needed to provision the aws_auth configmap in -kube-system. You can also use this provider to create your own kubernetes resources with the terraform kubernetes -provider. -Here's the gist of using it via the Terraform registry: ```hcl data "aws_eks_cluster" "cluster" { @@ -42,26 +38,18 @@ provider "kubernetes" { } module "my-cluster" { - source = "terraform-aws-modules/eks/aws" - cluster_name = "my-cluster" - subnets = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] - vpc_id = "vpc-1234556abcdef" + source = "terraform-aws-modules/eks/aws" + cluster_name = "my-cluster" + cluster_version = "1.14" + subnets = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + vpc_id = "vpc-1234556abcdef" worker_groups = [ { instance_type = "m4.large" asg_max_size = 5 - tags = [{ - key = "foo" - value = "bar" - propagate_at_launch = true - }] } ] - - tags = { - environment = "test" - } } ``` @@ -116,7 +104,7 @@ The [changelog](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/ ## Authors Created by [Brandon O'Connor](https://github.com/brandoconnor) - brandon@atscale.run. -Maintained by [Max Williams](https://github.com/max-rocket-internet) +Maintained by [Max Williams](https://github.com/max-rocket-internet) and [Thierno IB. BARRY](https://github.com/barryib). Many thanks to [the contributors listed here](https://github.com/terraform-aws-modules/terraform-aws-eks/graphs/contributors)! ## License diff --git a/docs/spot-instances.md b/docs/spot-instances.md index 4dcd8f1e85..06ca2e8068 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -87,6 +87,8 @@ Launch Template support is a recent addition to both AWS and this module. It mig ] ``` -## Important issues +## Important Notes -- https://github.com/kubernetes/autoscaler/issues/1133 +An issue with the cluster-autoscaler: https://github.com/kubernetes/autoscaler/issues/1133 + +AWS have released their own termination handler now: https://github.com/aws/aws-node-termination-handler diff --git a/docs/upgrading-to-aws-auth-kubernetes-provider.md b/docs/upgrading-to-aws-auth-kubernetes-provider.md deleted file mode 100644 index 345828b9d9..0000000000 --- a/docs/upgrading-to-aws-auth-kubernetes-provider.md +++ /dev/null @@ -1,14 +0,0 @@ -# Upgrading from version <= 7.x to 8.0.0 - -In version 8.0.0 the way the aws-auth config map in the kube-system namespaces is managed, has been changed. -Before this was managed via kubectl using a null resources. This was changed to be managed by the terraform kubernetes -provider. - -To upgrade you have to add the kubernetes provider to the place you are calling the module. You can see examples in -the [examples](../examples) folder. -You also have to delete the aws-auth config map before doing an apply. - -**This means you need to the apply with the same user/role that created the cluster.** - -Since this user will be the only one with admin on the k8s cluster. After that the resource is managed trough the -terraform kubernetes provider. \ No newline at end of file From cf3dcc5c58799f3b720924311e2a0574bc7553bb Mon Sep 17 00:00:00 2001 From: William Morgan Date: Wed, 4 Dec 2019 12:30:36 -0500 Subject: [PATCH 100/962] Support for EKS Managed Node Groups (#602) * Finished first cut of managed node groups * Updated formatting and extra fields. * Updating Changelog and README * Fixing formatting * Fixing docs. * Updating required Version * Updating changelog * Adding example for managed node groups * Managed IAM Roles for Nodegroups now have correct policies. Tags can now be added to node groups. * Fixing bug where people could set source_security_group_ids without setting ssh key causing a race condition within the aws provider. * Adding lifecycle create_before_destroy * Adding random pet names for create_before_destroy * Updating per comments. * Updating required versions of terraform * Updating per comments. * Updating vars * Updating minimum version for terraform * Change worker_groups_managed_node_groups to node_groups * Using for_each on the random_pet * Adding changes recommended by @eytanhanig * Update node_groups.tf --- .github/workflows/lint.yml | 2 +- CHANGELOG.md | 2 + README.md | 1 + examples/managed_node_groups/main.tf | 104 ++++++++++++++++++++++ examples/managed_node_groups/outputs.tf | 25 ++++++ examples/managed_node_groups/variables.tf | 52 +++++++++++ local.tf | 17 +++- node_groups.tf | 103 +++++++++++++++++++++ variables.tf | 6 ++ versions.tf | 4 +- 10 files changed, 311 insertions(+), 5 deletions(-) create mode 100644 examples/managed_node_groups/main.tf create mode 100644 examples/managed_node_groups/outputs.tf create mode 100644 examples/managed_node_groups/variables.tf create mode 100644 node_groups.tf diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a9eedc8aa9..c6511bd262 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -56,7 +56,7 @@ jobs: name: Minimum version check runs-on: ubuntu-latest container: - image: hashicorp/terraform:0.12.2 + image: hashicorp/terraform:0.12.6 steps: - uses: actions/checkout@master - name: Validate Code diff --git a/CHANGELOG.md b/CHANGELOG.md index abee4af9cc..0fb7bfccf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,14 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v7.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...HEAD)] - 2019-??-??] - Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) +- Support for AWS EKS Managed Node Groups. (by @wmorgan6796) - Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) - **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. Read the [docs](docs/upgrading-to-aws-auth-kubernetes-provider.md) for more info (by @sdehaes) - Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) - Exit with error code when `aws-auth` configmap is unable to be updated (by @knittingdev) - Fix deprecated interpolation-only expression (by @angelabad) - Fix broken terraform plan/apply on a cluster < 1.14 (by @hodduc) +- Updated required version of AWS Provider to >= v2.38.0 for Managed Node Groups (by @wmorgan6796) #### Important notes diff --git a/README.md b/README.md index 6e534eae84..76a385f329 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | | map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | | map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | +| node\_groups | A list of maps defining node group configurations to be defined using AWS EKS Managed Node Groups. See workers_group_defaults for valid keys. | any | `[]` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `"null"` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | | tags | A map of tags to add to all resources. | map(string) | `{}` | no | diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf new file mode 100644 index 0000000000..00b93aa9ff --- /dev/null +++ b/examples/managed_node_groups/main.tf @@ -0,0 +1,104 @@ +terraform { + required_version = ">= 0.12.6" +} + +provider "aws" { + version = ">= 2.28.1" + region = var.region +} + +provider "random" { + version = "~> 2.1" +} + +provider "local" { + version = "~> 1.2" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + +data "aws_availability_zones" "available" { +} + +locals { + cluster_name = "test-eks-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 2.6" + + name = "test-vpc" + cidr = "172.16.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"] + public_subnets = ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } +} + +module "eks" { + source = "../.." + cluster_name = local.cluster_name + subnets = module.vpc.private_subnets + + tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + vpc_id = module.vpc.vpc_id + + node_groups = [ + { + name = "example" + + node_group_desired_capacity = 1 + node_group_max_capacity = 10 + node_group_min_capacity = 1 + + instance_type = "m5.large" + node_group_k8s_labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + node_group_additional_tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + } + ] + + map_roles = var.map_roles + map_users = var.map_users + map_accounts = var.map_accounts +} diff --git a/examples/managed_node_groups/outputs.tf b/examples/managed_node_groups/outputs.tf new file mode 100644 index 0000000000..a0788aff1d --- /dev/null +++ b/examples/managed_node_groups/outputs.tf @@ -0,0 +1,25 @@ +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = module.eks.cluster_security_group_id +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = module.eks.kubeconfig +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = module.eks.config_map_aws_auth +} + +output "region" { + description = "AWS region." + value = var.region +} + diff --git a/examples/managed_node_groups/variables.tf b/examples/managed_node_groups/variables.tf new file mode 100644 index 0000000000..7085aeabd4 --- /dev/null +++ b/examples/managed_node_groups/variables.tf @@ -0,0 +1,52 @@ +variable "region" { + default = "us-west-2" +} + +variable "map_accounts" { + description = "Additional AWS account numbers to add to the aws-auth configmap." + type = list(string) + + default = [ + "777777777777", + "888888888888", + ] +} + +variable "map_roles" { + description = "Additional IAM roles to add to the aws-auth configmap." + type = list(object({ + rolearn = string + username = string + groups = list(string) + })) + + default = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] +} + +variable "map_users" { + description = "Additional IAM users to add to the aws-auth configmap." + type = list(object({ + userarn = string + username = string + groups = list(string) + })) + + default = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] +} diff --git a/local.tf b/local.tf index d3eccdbad2..ba0bb72f2d 100644 --- a/local.tf +++ b/local.tf @@ -16,8 +16,9 @@ locals { default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name - worker_group_count = length(var.worker_groups) - worker_group_launch_template_count = length(var.worker_groups_launch_template) + worker_group_count = length(var.worker_groups) + worker_group_launch_template_count = length(var.worker_groups_launch_template) + worker_group_managed_node_group_count = length(var.node_groups) default_ami_id_linux = data.aws_ami.eks_worker.id default_ami_id_windows = data.aws_ami.eks_worker_windows.id @@ -79,6 +80,15 @@ locals { spot_allocation_strategy = "lowest-price" # Valid options are 'lowest-price' and 'capacity-optimized'. If 'lowest-price', the Auto Scaling group launches instances using the Spot pools with the lowest price, and evenly allocates your instances across the number of Spot pools. If 'capacity-optimized', the Auto Scaling group launches instances using Spot pools that are optimally chosen based on the available Spot capacity. spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price + ami_type = "AL2_x86_64" # AMI Type to use for the Managed Node Groups. Can be either: AL2_x86_64 or AL2_x86_64_GPU + ami_release_version = "" # AMI Release Version of the Managed Node Groups + source_security_group_id = [] # Source Security Group IDs to allow SSH Access to the Nodes. NOTE: IF LEFT BLANK, AND A KEY IS SPECIFIED, THE SSH PORT WILL BE OPENNED TO THE WORLD + node_group_k8s_labels = {} # Kubernetes Labels to apply to the nodes within the Managed Node Group + node_group_desired_capacity = 1 # Desired capacity of the Node Group + node_group_min_capacity = 1 # Min capacity of the Node Group (Minimum value allowed is 1) + node_group_max_capacity = 3 # Max capacity of the Node Group + node_group_iam_role_arn = "" # IAM role to use for Managed Node Groups instead of default one created by the automation + node_group_additional_tags = {} # Additional tags to be applied to the Node Groups } workers_group_defaults = merge( @@ -123,4 +133,7 @@ locals { "t2.small", "t2.xlarge" ] + + node_groups = { for node_group in var.node_groups : node_group["name"] => node_group } + } diff --git a/node_groups.tf b/node_groups.tf new file mode 100644 index 0000000000..c47436fd03 --- /dev/null +++ b/node_groups.tf @@ -0,0 +1,103 @@ +resource "aws_iam_role" "node_groups" { + count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + name = "${var.workers_role_name != "" ? var.workers_role_name : aws_eks_cluster.this.name}-managed-node-groups" + assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json + permissions_boundary = var.permissions_boundary + path = var.iam_path + force_detach_policies = true + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "node_groups_AmazonEKSWorkerNodePolicy" { + count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" + role = aws_iam_role.node_groups[0].name +} + +resource "aws_iam_role_policy_attachment" "node_groups_AmazonEKS_CNI_Policy" { + count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" + role = aws_iam_role.node_groups[0].name +} + +resource "aws_iam_role_policy_attachment" "node_groups_AmazonEC2ContainerRegistryReadOnly" { + count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + role = aws_iam_role.node_groups[0].name +} + +resource "aws_iam_role_policy_attachment" "node_groups_additional_policies" { + for_each = toset(var.workers_additional_policies) + + role = aws_iam_role.node_groups[0].name + policy_arn = each.key +} + +resource "aws_iam_role_policy_attachment" "node_groups_autoscaling" { + count = var.manage_worker_autoscaling_policy && var.attach_worker_autoscaling_policy && local.worker_group_managed_node_group_count > 0 ? 1 : 0 + policy_arn = aws_iam_policy.node_groups_autoscaling[0].arn + role = aws_iam_role.node_groups[0].name +} + +resource "aws_iam_policy" "node_groups_autoscaling" { + count = var.manage_worker_autoscaling_policy && local.worker_group_managed_node_group_count > 0 ? 1 : 0 + name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this.name}" + description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this.name}" + policy = data.aws_iam_policy_document.worker_autoscaling.json + path = var.iam_path +} + +resource "random_pet" "node_groups" { + for_each = local.node_groups + + separator = "-" + length = 2 + + keepers = { + instance_type = lookup(each.value, "instance_type", local.workers_group_defaults["instance_type"]) + + ec2_ssh_key = lookup(each.value, "key_name", local.workers_group_defaults["key_name"]) + + source_security_group_ids = join("-", compact( + lookup(each.value, "source_security_group_ids", local.workers_group_defaults["source_security_group_id"] + ))) + + node_group_name = join("-", [var.cluster_name, each.value["name"]]) + } +} + +resource "aws_eks_node_group" "workers" { + for_each = local.node_groups + + node_group_name = join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id]) + + cluster_name = var.cluster_name + node_role_arn = lookup(each.value, "iam_role_arn", aws_iam_role.node_groups[0].arn) + subnet_ids = lookup(each.value, "subnets", local.workers_group_defaults["subnets"]) + + scaling_config { + desired_size = lookup(each.value, "node_group_desired_capacity", local.workers_group_defaults["asg_desired_capacity"]) + max_size = lookup(each.value, "node_group_max_capacity", local.workers_group_defaults["asg_max_size"]) + min_size = lookup(each.value, "node_group_min_capacity", local.workers_group_defaults["asg_min_size"]) + } + + ami_type = lookup(each.value, "ami_type", null) + disk_size = lookup(each.value, "root_volume_size", null) + instance_types = [lookup(each.value, "instance_type", null)] + labels = lookup(each.value, "node_group_k8s_labels", null) + release_version = lookup(each.value, "ami_release_version", null) + + # This sometimes breaks idempotency as described in https://github.com/terraform-providers/terraform-provider-aws/issues/11063 + remote_access { + ec2_ssh_key = lookup(each.value, "key_name", "") != "" ? each.value["key_name"] : null + source_security_group_ids = lookup(each.value, "key_name", "") != "" ? lookup(each.value, "source_security_group_ids", []) : null + } + + version = aws_eks_cluster.this.version + + tags = lookup(each.value, "node_group_additional_tags", null) + + lifecycle { + create_before_destroy = true + } +} diff --git a/variables.tf b/variables.tf index 06c9460fb8..498313b886 100644 --- a/variables.tf +++ b/variables.tf @@ -293,3 +293,9 @@ variable "attach_worker_cni_policy" { type = bool default = true } + +variable "node_groups" { + description = "A list of maps defining node group configurations to be defined using AWS EKS Managed Node Groups. See workers_group_defaults for valid keys." + type = any + default = [] +} \ No newline at end of file diff --git a/versions.tf b/versions.tf index 0a6046ef7f..61c59ea9e6 100644 --- a/versions.tf +++ b/versions.tf @@ -1,8 +1,8 @@ terraform { - required_version = ">= 0.12.2" + required_version = ">= 0.12.6" required_providers { - aws = ">= 2.31.0" + aws = ">= 2.38.0" local = ">= 1.2" null = ">= 2.1" template = ">= 2.1" From 9de5b53cb486415d874fc5433a876a4e569e052a Mon Sep 17 00:00:00 2001 From: Shan K Date: Wed, 4 Dec 2019 23:16:44 +0530 Subject: [PATCH 101/962] remove empty [] to mapRoles object in aws-auth (#606) * remove empty [] to mapRoles object in aws-auth Simply having ${yamlencode(var.map_roles)} in mapRoles for aws-auth creates a empty [] at the end after adding the default roles. Changing it to be added only when its not empty * Update aws_auth.tf --- aws_auth.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_auth.tf b/aws_auth.tf index 263ceab6e8..9e9d48f259 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -53,7 +53,7 @@ resource "kubernetes_config_map" "aws_auth" { data = { mapRoles = < Date: Thu, 5 Dec 2019 21:24:13 +0530 Subject: [PATCH 102/962] Fix PR merge: #606 (#611) * Update aws_auth.tf * aws-auth config map --- CHANGELOG.md | 1 + aws_auth.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fb7bfccf7..e0223a2e3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) - Support for AWS EKS Managed Node Groups. (by @wmorgan6796) - Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) +- Added a if check on `aws-auth` configmap when map_roles is empty. - **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. Read the [docs](docs/upgrading-to-aws-auth-kubernetes-provider.md) for more info (by @sdehaes) - Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) - Exit with error code when `aws-auth` configmap is unable to be updated (by @knittingdev) diff --git a/aws_auth.tf b/aws_auth.tf index 9e9d48f259..916661dd64 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -53,7 +53,7 @@ resource "kubernetes_config_map" "aws_auth" { data = { mapRoles = < Date: Mon, 9 Dec 2019 14:26:55 +0530 Subject: [PATCH 103/962] remove empty [] to mapRoles in aws-auth (revised) (#619) --- aws_auth.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_auth.tf b/aws_auth.tf index 916661dd64..7dd80897f3 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -53,7 +53,7 @@ resource "kubernetes_config_map" "aws_auth" { data = { mapRoles = < Date: Mon, 9 Dec 2019 09:59:26 +0100 Subject: [PATCH 104/962] Bump minimum Terraform to 0.12.9 (#617) --- .github/workflows/lint.yml | 2 +- CHANGELOG.md | 1 + versions.tf | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c6511bd262..0c71b11d42 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -56,7 +56,7 @@ jobs: name: Minimum version check runs-on: ubuntu-latest container: - image: hashicorp/terraform:0.12.6 + image: hashicorp/terraform:0.12.9 steps: - uses: actions/checkout@master - name: Validate Code diff --git a/CHANGELOG.md b/CHANGELOG.md index e0223a2e3e..b167b40d03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fix deprecated interpolation-only expression (by @angelabad) - Fix broken terraform plan/apply on a cluster < 1.14 (by @hodduc) - Updated required version of AWS Provider to >= v2.38.0 for Managed Node Groups (by @wmorgan6796) +- Updated minimum version of Terraform to avoid a bug (by @dpiddockcmp) #### Important notes diff --git a/versions.tf b/versions.tf index 61c59ea9e6..06a6c1cc50 100644 --- a/versions.tf +++ b/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.12.6" + required_version = ">= 0.12.9" required_providers { aws = ">= 2.38.0" From 124ea7c15154d4c7c0beb71232d15a494a1468e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomislav=20Toma=C5=A1i=C4=87?= Date: Mon, 9 Dec 2019 09:06:10 +0000 Subject: [PATCH 105/962] Add destroy time flag (#580) * Add destroy-time flag * Update changelog Fix cluster count * Fix cluster count * Fix docs * Fix outputs * Fix unsupported attribute on cluster_certificate_authority_data output Co-Authored-By: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> * Remove unnecessary flatten from cluster_endpoint output Co-Authored-By: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> * Improve description of var.enabled * Fix errors manifesting when used on an existing-cluster * Update README.md * Renamed destroy-time flag * Revert removal of changelog addition entry * Update flag name in readme * Update flag variable name * Update cluster referencing for consistency * Update flag name to `create_eks` * Fixed incorrect count-based reference to aws_eks_cluster.this (there's only one) * Replaced all incorrect aws_eks_cluster.this[count.index] references (there will be just one, so using '[0]'). * Changelog update, explicitly mentioning flag * Fixed interpolation deprecation warning * Fixed outputs to support conditional cluster * Applied create_eks to aws_auth.tf * Removed unused variable. Updated Changelog. Formatting. * Fixed references to aws_eks_cluster.this[0] that would raise errors when setting create_eks to false whilst having launch templates or launch configurations configured. * Readme and example updates. * Revert "Readme and example updates." This reverts commit 18a0746355e136010ad54858a1b518406f6a3638. * Updated readme section of conditionally creation with provider example. * Added conditions to node_groups. * Fixed reversed map_roles check * Update aws_auth.tf Revert this due to https://github.com/terraform-aws-modules/terraform-aws-eks/pull/611 --- CHANGELOG.md | 4 ++- README.md | 36 +++++++++++++++++++- aws_auth.tf | 6 ++-- cluster.tf | 15 +++++---- data.tf | 23 ++++++------- kubectl.tf | 5 ++- node_groups.tf | 28 ++++++++-------- outputs.tf | 16 ++++----- variables.tf | 14 ++++---- workers.tf | 67 +++++++++++++++++++------------------- workers_launch_template.tf | 24 +++++++------- 11 files changed, 138 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b167b40d03..e40caa423a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,15 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v7.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...HEAD)] - 2019-??-??] +## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...HEAD)] - 2019-??-??] - Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) +- Added flag `create_eks` to conditionally create resources (by @syst0m / @tbeijen) - Support for AWS EKS Managed Node Groups. (by @wmorgan6796) - Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) - Added a if check on `aws-auth` configmap when map_roles is empty. - **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. Read the [docs](docs/upgrading-to-aws-auth-kubernetes-provider.md) for more info (by @sdehaes) +- Removed no longer used variable `write_aws_auth_config` (by @tbeijen) - Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) - Exit with error code when `aws-auth` configmap is unable to be updated (by @knittingdev) - Fix deprecated interpolation-only expression (by @angelabad) diff --git a/README.md b/README.md index 76a385f329..20d2501f27 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,40 @@ module "my-cluster" { ] } ``` +## Conditional creation + +Sometimes you need to have a way to create EKS resources conditionally but Terraform does not allow to use `count` inside `module` block, so the solution is to specify argument `create_eks`. + +Using this feature _and_ having `manage_aws_auth=true` (the default) requires to set up the kubernetes provider in a way that allows the data sources to not exist. + +```hcl +data "aws_eks_cluster" "cluster" { + count = var.create_eks ? 1 : 0 + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + count = var.create_eks ? 1 : 0 + name = module.eks.cluster_id +} + +# In case of not creating the cluster, this will be an incompletely configured, unused provider, which poses no problem. +provider "kubernetes" { + host = element(concat(data.aws_eks_cluster.cluster[*].endpoint, list("")), 0) + cluster_ca_certificate = base64decode(element(concat(data.aws_eks_cluster.cluster[*].certificate_authority.0.data, list("")), 0)) + token = element(concat(data.aws_eks_cluster_auth.cluster[*].token, list("")), 0) + load_config_file = false + version = "~> 1.10" +} + +# This cluster will not be created +module "eks" { + source = "terraform-aws-modules/eks/aws" + + create_eks = false + # ... omitted +} +``` ## Other documentation @@ -131,6 +165,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | string | `""` | no | | cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.14"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | string | `"./"` | no | +| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | bool | `"true"` | no | | iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | | kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | | kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | string | `"aws-iam-authenticator"` | no | @@ -164,7 +199,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | workers\_additional\_policies | Additional policies to be added to workers | list(string) | `[]` | no | | workers\_group\_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | any | `{}` | no | | workers\_role\_name | User defined workers role name. | string | `""` | no | -| write\_aws\_auth\_config | Whether to write the aws-auth configmap file. | bool | `"true"` | no | | write\_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | bool | `"true"` | no | ## Outputs diff --git a/aws_auth.tf b/aws_auth.tf index 7dd80897f3..aabf6fdb94 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -2,7 +2,7 @@ data "aws_caller_identity" "current" { } data "template_file" "launch_template_worker_role_arns" { - count = local.worker_group_launch_template_count + count = var.create_eks ? local.worker_group_launch_template_count : 0 template = file("${path.module}/templates/worker-role.tpl") vars = { @@ -22,7 +22,7 @@ data "template_file" "launch_template_worker_role_arns" { } data "template_file" "worker_role_arns" { - count = local.worker_group_count + count = var.create_eks ? local.worker_group_count : 0 template = file("${path.module}/templates/worker-role.tpl") vars = { @@ -43,7 +43,7 @@ data "template_file" "worker_role_arns" { } resource "kubernetes_config_map" "aws_auth" { - count = var.manage_aws_auth ? 1 : 0 + count = var.create_eks && var.manage_aws_auth ? 1 : 0 metadata { name = "aws-auth" diff --git a/cluster.tf b/cluster.tf index 7b8a1c9184..b7ced5c144 100644 --- a/cluster.tf +++ b/cluster.tf @@ -1,5 +1,5 @@ resource "aws_cloudwatch_log_group" "this" { - count = length(var.cluster_enabled_log_types) > 0 ? 1 : 0 + count = length(var.cluster_enabled_log_types) > 0 && var.create_eks ? 1 : 0 name = "/aws/eks/${var.cluster_name}/cluster" retention_in_days = var.cluster_log_retention_in_days kms_key_id = var.cluster_log_kms_key_id @@ -7,6 +7,7 @@ resource "aws_cloudwatch_log_group" "this" { } resource "aws_eks_cluster" "this" { + count = var.create_eks ? 1 : 0 name = var.cluster_name enabled_cluster_log_types = var.cluster_enabled_log_types role_arn = local.cluster_iam_role_arn @@ -33,7 +34,7 @@ resource "aws_eks_cluster" "this" { } resource "aws_security_group" "cluster" { - count = var.cluster_create_security_group ? 1 : 0 + count = var.cluster_create_security_group && var.create_eks ? 1 : 0 name_prefix = var.cluster_name description = "EKS cluster security group." vpc_id = var.vpc_id @@ -46,7 +47,7 @@ resource "aws_security_group" "cluster" { } resource "aws_security_group_rule" "cluster_egress_internet" { - count = var.cluster_create_security_group ? 1 : 0 + count = var.cluster_create_security_group && var.create_eks ? 1 : 0 description = "Allow cluster egress access to the Internet." protocol = "-1" security_group_id = local.cluster_security_group_id @@ -57,7 +58,7 @@ resource "aws_security_group_rule" "cluster_egress_internet" { } resource "aws_security_group_rule" "cluster_https_worker_ingress" { - count = var.cluster_create_security_group ? 1 : 0 + count = var.cluster_create_security_group && var.create_eks ? 1 : 0 description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" security_group_id = local.cluster_security_group_id @@ -68,7 +69,7 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { } resource "aws_iam_role" "cluster" { - count = var.manage_cluster_iam_resources ? 1 : 0 + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 name_prefix = var.cluster_name assume_role_policy = data.aws_iam_policy_document.cluster_assume_role_policy.json permissions_boundary = var.permissions_boundary @@ -78,13 +79,13 @@ resource "aws_iam_role" "cluster" { } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" { - count = var.manage_cluster_iam_resources ? 1 : 0 + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" role = local.cluster_iam_role_name } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { - count = var.manage_cluster_iam_resources ? 1 : 0 + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" role = local.cluster_iam_role_name } diff --git a/data.tf b/data.tf index 20fd35674b..f43ac9e680 100644 --- a/data.tf +++ b/data.tf @@ -66,19 +66,20 @@ data "aws_iam_policy_document" "cluster_assume_role_policy" { } data "template_file" "kubeconfig" { + count = var.create_eks ? 1 : 0 template = file("${path.module}/templates/kubeconfig.tpl") vars = { kubeconfig_name = local.kubeconfig_name - endpoint = aws_eks_cluster.this.endpoint - cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data + endpoint = aws_eks_cluster.this[0].endpoint + cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data aws_authenticator_command = var.kubeconfig_aws_authenticator_command aws_authenticator_command_args = length(var.kubeconfig_aws_authenticator_command_args) > 0 ? " - ${join( "\n - ", var.kubeconfig_aws_authenticator_command_args, )}" : " - ${join( "\n - ", - formatlist("\"%s\"", ["token", "-i", aws_eks_cluster.this.name]), + formatlist("\"%s\"", ["token", "-i", aws_eks_cluster.this[0].name]), )}" aws_authenticator_additional_args = length(var.kubeconfig_aws_authenticator_additional_args) > 0 ? " - ${join( "\n - ", @@ -107,7 +108,7 @@ EOF } data "template_file" "userdata" { - count = local.worker_group_count + count = var.create_eks ? local.worker_group_count : 0 template = lookup( var.worker_groups[count.index], "userdata_template_file", @@ -120,9 +121,9 @@ data "template_file" "userdata" { vars = merge({ platform = lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) - cluster_name = aws_eks_cluster.this.name - endpoint = aws_eks_cluster.this.endpoint - cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data + cluster_name = aws_eks_cluster.this[0].name + endpoint = aws_eks_cluster.this[0].endpoint + cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data pre_userdata = lookup( var.worker_groups[count.index], "pre_userdata", @@ -153,7 +154,7 @@ data "template_file" "userdata" { } data "template_file" "launch_template_userdata" { - count = local.worker_group_launch_template_count + count = var.create_eks ? local.worker_group_launch_template_count : 0 template = lookup( var.worker_groups_launch_template[count.index], "userdata_template_file", @@ -166,9 +167,9 @@ data "template_file" "launch_template_userdata" { vars = merge({ platform = lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) - cluster_name = aws_eks_cluster.this.name - endpoint = aws_eks_cluster.this.endpoint - cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data + cluster_name = aws_eks_cluster.this[0].name + endpoint = aws_eks_cluster.this[0].endpoint + cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data pre_userdata = lookup( var.worker_groups_launch_template[count.index], "pre_userdata", diff --git a/kubectl.tf b/kubectl.tf index c7b86d4698..56aba1ffd9 100644 --- a/kubectl.tf +++ b/kubectl.tf @@ -1,6 +1,5 @@ resource "local_file" "kubeconfig" { - count = var.write_kubeconfig ? 1 : 0 - content = data.template_file.kubeconfig.rendered + count = var.write_kubeconfig && var.create_eks ? 1 : 0 + content = data.template_file.kubeconfig[0].rendered filename = substr(var.config_output_path, -1, 1) == "/" ? "${var.config_output_path}kubeconfig_${var.cluster_name}" : var.config_output_path } - diff --git a/node_groups.tf b/node_groups.tf index c47436fd03..c604da6e0c 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -1,6 +1,6 @@ resource "aws_iam_role" "node_groups" { - count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 - name = "${var.workers_role_name != "" ? var.workers_role_name : aws_eks_cluster.this.name}-managed-node-groups" + count = var.create_eks && local.worker_group_managed_node_group_count > 0 ? 1 : 0 + name = "${var.workers_role_name != "" ? var.workers_role_name : aws_eks_cluster.this[0].name}-managed-node-groups" assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json permissions_boundary = var.permissions_boundary path = var.iam_path @@ -9,46 +9,46 @@ resource "aws_iam_role" "node_groups" { } resource "aws_iam_role_policy_attachment" "node_groups_AmazonEKSWorkerNodePolicy" { - count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + count = var.create_eks && local.worker_group_managed_node_group_count > 0 ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" role = aws_iam_role.node_groups[0].name } resource "aws_iam_role_policy_attachment" "node_groups_AmazonEKS_CNI_Policy" { - count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + count = var.create_eks && local.worker_group_managed_node_group_count > 0 ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" role = aws_iam_role.node_groups[0].name } resource "aws_iam_role_policy_attachment" "node_groups_AmazonEC2ContainerRegistryReadOnly" { - count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + count = var.create_eks && local.worker_group_managed_node_group_count > 0 ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" role = aws_iam_role.node_groups[0].name } resource "aws_iam_role_policy_attachment" "node_groups_additional_policies" { - for_each = toset(var.workers_additional_policies) + for_each = var.create_eks && local.worker_group_managed_node_group_count > 0 ? toset(var.workers_additional_policies) : [] role = aws_iam_role.node_groups[0].name policy_arn = each.key } resource "aws_iam_role_policy_attachment" "node_groups_autoscaling" { - count = var.manage_worker_autoscaling_policy && var.attach_worker_autoscaling_policy && local.worker_group_managed_node_group_count > 0 ? 1 : 0 + count = var.create_eks && var.manage_worker_autoscaling_policy && var.attach_worker_autoscaling_policy && local.worker_group_managed_node_group_count > 0 ? 1 : 0 policy_arn = aws_iam_policy.node_groups_autoscaling[0].arn role = aws_iam_role.node_groups[0].name } resource "aws_iam_policy" "node_groups_autoscaling" { - count = var.manage_worker_autoscaling_policy && local.worker_group_managed_node_group_count > 0 ? 1 : 0 - name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this.name}" - description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this.name}" - policy = data.aws_iam_policy_document.worker_autoscaling.json + count = var.create_eks && var.manage_worker_autoscaling_policy && local.worker_group_managed_node_group_count > 0 ? 1 : 0 + name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this[0].name}" + description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this[0].name}" + policy = data.aws_iam_policy_document.worker_autoscaling[0].json path = var.iam_path } resource "random_pet" "node_groups" { - for_each = local.node_groups + for_each = var.create_eks ? local.node_groups : {} separator = "-" length = 2 @@ -67,7 +67,7 @@ resource "random_pet" "node_groups" { } resource "aws_eks_node_group" "workers" { - for_each = local.node_groups + for_each = var.create_eks ? local.node_groups : {} node_group_name = join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id]) @@ -93,7 +93,7 @@ resource "aws_eks_node_group" "workers" { source_security_group_ids = lookup(each.value, "key_name", "") != "" ? lookup(each.value, "source_security_group_ids", []) : null } - version = aws_eks_cluster.this.version + version = aws_eks_cluster.this[0].version tags = lookup(each.value, "node_group_additional_tags", null) diff --git a/outputs.tf b/outputs.tf index d18dc783b8..2478dc2acb 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,26 +1,26 @@ output "cluster_id" { description = "The name/id of the EKS cluster." - value = aws_eks_cluster.this.id + value = element(concat(aws_eks_cluster.this.*.id, list("")), 0) } output "cluster_arn" { description = "The Amazon Resource Name (ARN) of the cluster." - value = aws_eks_cluster.this.arn + value = element(concat(aws_eks_cluster.this.*.arn, list("")), 0) } output "cluster_certificate_authority_data" { description = "Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster." - value = aws_eks_cluster.this.certificate_authority[0].data + value = element(concat(aws_eks_cluster.this[*].certificate_authority[0].data, list("")), 0) } output "cluster_endpoint" { description = "The endpoint for your EKS Kubernetes API." - value = aws_eks_cluster.this.endpoint + value = element(concat(aws_eks_cluster.this.*.endpoint, list("")), 0) } output "cluster_version" { description = "The Kubernetes server version for the EKS cluster." - value = aws_eks_cluster.this.version + value = element(concat(aws_eks_cluster.this[*].version, list("")), 0) } output "cluster_security_group_id" { @@ -45,17 +45,17 @@ output "cluster_iam_role_arn" { output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster OIDC Issuer" - value = concat(aws_eks_cluster.this.identity.*.oidc.0.issuer, [""])[0] + value = concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""])[0] } output "cloudwatch_log_group_name" { description = "Name of cloudwatch log group created" - value = aws_cloudwatch_log_group.this.*.name + value = aws_cloudwatch_log_group.this[*].name } output "kubeconfig" { description = "kubectl config file contents for this EKS cluster." - value = data.template_file.kubeconfig.rendered + value = concat(data.template_file.kubeconfig[*].rendered, [""])[0] } output "kubeconfig_filename" { diff --git a/variables.tf b/variables.tf index 498313b886..366c9267f6 100644 --- a/variables.tf +++ b/variables.tf @@ -48,12 +48,6 @@ variable "manage_aws_auth" { default = true } -variable "write_aws_auth_config" { - description = "Whether to write the aws-auth configmap file." - type = bool - default = true -} - variable "map_accounts" { description = "Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format." type = list(string) @@ -294,8 +288,14 @@ variable "attach_worker_cni_policy" { default = true } +variable "create_eks" { + description = "Controls if EKS resources should be created (it affects almost all resources)" + type = bool + default = true +} + variable "node_groups" { description = "A list of maps defining node group configurations to be defined using AWS EKS Managed Node Groups. See workers_group_defaults for valid keys." type = any default = [] -} \ No newline at end of file +} diff --git a/workers.tf b/workers.tf index e946e78e15..ce0f1e2277 100644 --- a/workers.tf +++ b/workers.tf @@ -1,12 +1,12 @@ # Worker Groups using Launch Configurations resource "aws_autoscaling_group" "workers" { - count = local.worker_group_count + count = var.create_eks ? local.worker_group_count : 0 name_prefix = join( "-", compact( [ - aws_eks_cluster.this.name, + aws_eks_cluster.this[0].name, lookup(var.worker_groups[count.index], "name", count.index), lookup(var.worker_groups[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers[count.index].id : "" ] @@ -91,16 +91,16 @@ resource "aws_autoscaling_group" "workers" { [ { "key" = "Name" - "value" = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg" + "value" = "${aws_eks_cluster.this[0].name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg" "propagate_at_launch" = true }, { - "key" = "kubernetes.io/cluster/${aws_eks_cluster.this.name}" + "key" = "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" "value" = "owned" "propagate_at_launch" = true }, { - "key" = "k8s.io/cluster/${aws_eks_cluster.this.name}" + "key" = "k8s.io/cluster/${aws_eks_cluster.this[0].name}" "value" = "owned" "propagate_at_launch" = true }, @@ -114,8 +114,8 @@ resource "aws_autoscaling_group" "workers" { "propagate_at_launch" = false }, { - "key" = "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}" - "value" = aws_eks_cluster.this.name + "key" = "k8s.io/cluster-autoscaler/${aws_eks_cluster.this[0].name}" + "value" = aws_eks_cluster.this[0].name "propagate_at_launch" = false }, { @@ -143,8 +143,8 @@ resource "aws_autoscaling_group" "workers" { } resource "aws_launch_configuration" "workers" { - count = local.worker_group_count - name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}" + count = var.create_eks ? local.worker_group_count : 0 + name_prefix = "${aws_eks_cluster.this[0].name}-${lookup(var.worker_groups[count.index], "name", count.index)}" associate_public_ip_address = lookup( var.worker_groups[count.index], "public_ip", @@ -232,7 +232,7 @@ resource "aws_launch_configuration" "workers" { } resource "random_pet" "workers" { - count = local.worker_group_count + count = var.create_eks ? local.worker_group_count : 0 separator = "-" length = 2 @@ -243,21 +243,21 @@ resource "random_pet" "workers" { } resource "aws_security_group" "workers" { - count = var.worker_create_security_group ? 1 : 0 - name_prefix = aws_eks_cluster.this.name + count = var.worker_create_security_group && var.create_eks ? 1 : 0 + name_prefix = aws_eks_cluster.this[0].name description = "Security group for all nodes in the cluster." vpc_id = var.vpc_id tags = merge( var.tags, { - "Name" = "${aws_eks_cluster.this.name}-eks_worker_sg" - "kubernetes.io/cluster/${aws_eks_cluster.this.name}" = "owned" + "Name" = "${aws_eks_cluster.this[0].name}-eks_worker_sg" + "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" = "owned" }, ) } resource "aws_security_group_rule" "workers_egress_internet" { - count = var.worker_create_security_group ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow nodes all egress to the Internet." protocol = "-1" security_group_id = local.worker_security_group_id @@ -268,7 +268,7 @@ resource "aws_security_group_rule" "workers_egress_internet" { } resource "aws_security_group_rule" "workers_ingress_self" { - count = var.worker_create_security_group ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow node to communicate with each other." protocol = "-1" security_group_id = local.worker_security_group_id @@ -279,7 +279,7 @@ resource "aws_security_group_rule" "workers_ingress_self" { } resource "aws_security_group_rule" "workers_ingress_cluster" { - count = var.worker_create_security_group ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow workers pods to receive communication from the cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id @@ -290,7 +290,7 @@ resource "aws_security_group_rule" "workers_ingress_cluster" { } resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { - count = var.worker_create_security_group ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 + count = var.worker_create_security_group && var.create_eks ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 description = "Allow workers Kubelets to receive communication from the cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id @@ -301,7 +301,7 @@ resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { } resource "aws_security_group_rule" "workers_ingress_cluster_https" { - count = var.worker_create_security_group ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id @@ -312,8 +312,8 @@ resource "aws_security_group_rule" "workers_ingress_cluster_https" { } resource "aws_iam_role" "workers" { - count = var.manage_worker_iam_resources ? 1 : 0 - name_prefix = var.workers_role_name != "" ? null : aws_eks_cluster.this.name + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + name_prefix = var.workers_role_name != "" ? null : aws_eks_cluster.this[0].name name = var.workers_role_name != "" ? var.workers_role_name : null assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json permissions_boundary = var.permissions_boundary @@ -323,8 +323,8 @@ resource "aws_iam_role" "workers" { } resource "aws_iam_instance_profile" "workers" { - count = var.manage_worker_iam_resources ? local.worker_group_count : 0 - name_prefix = aws_eks_cluster.this.name + count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_count : 0 + name_prefix = aws_eks_cluster.this[0].name role = lookup( var.worker_groups[count.index], "iam_role_id", @@ -335,44 +335,45 @@ resource "aws_iam_instance_profile" "workers" { } resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { - count = var.manage_worker_iam_resources ? 1 : 0 + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { - count = var.manage_worker_iam_resources && var.attach_worker_cni_policy ? 1 : 0 + count = var.manage_worker_iam_resources && var.attach_worker_cni_policy && var.create_eks ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" { - count = var.manage_worker_iam_resources ? 1 : 0 + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_additional_policies" { - count = var.manage_worker_iam_resources ? length(var.workers_additional_policies) : 0 + count = var.manage_worker_iam_resources && var.create_eks ? length(var.workers_additional_policies) : 0 role = aws_iam_role.workers[0].name policy_arn = var.workers_additional_policies[count.index] } resource "aws_iam_role_policy_attachment" "workers_autoscaling" { - count = var.manage_worker_iam_resources && var.manage_worker_autoscaling_policy && var.attach_worker_autoscaling_policy ? 1 : 0 + count = var.manage_worker_iam_resources && var.manage_worker_autoscaling_policy && var.attach_worker_autoscaling_policy && var.create_eks ? 1 : 0 policy_arn = aws_iam_policy.worker_autoscaling[0].arn role = aws_iam_role.workers[0].name } resource "aws_iam_policy" "worker_autoscaling" { - count = var.manage_worker_iam_resources && var.manage_worker_autoscaling_policy ? 1 : 0 - name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this.name}" - description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this.name}" - policy = data.aws_iam_policy_document.worker_autoscaling.json + count = var.manage_worker_iam_resources && var.manage_worker_autoscaling_policy && var.create_eks ? 1 : 0 + name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this[0].name}" + description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this[0].name}" + policy = data.aws_iam_policy_document.worker_autoscaling[0].json path = var.iam_path } data "aws_iam_policy_document" "worker_autoscaling" { + count = var.manage_worker_iam_resources && var.manage_worker_autoscaling_policy && var.create_eks ? 1 : 0 statement { sid = "eksWorkerAutoscalingAll" effect = "Allow" @@ -402,7 +403,7 @@ data "aws_iam_policy_document" "worker_autoscaling" { condition { test = "StringEquals" - variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${aws_eks_cluster.this.name}" + variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" values = ["owned"] } diff --git a/workers_launch_template.tf b/workers_launch_template.tf index fefacd63cb..519a289942 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -1,12 +1,12 @@ # Worker Groups using Launch Templates resource "aws_autoscaling_group" "workers_launch_template" { - count = local.worker_group_launch_template_count + count = var.create_eks ? local.worker_group_launch_template_count : 0 name_prefix = join( "-", compact( [ - aws_eks_cluster.this.name, + aws_eks_cluster.this[0].name, lookup(var.worker_groups_launch_template[count.index], "name", count.index), lookup(var.worker_groups_launch_template[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers_launch_template[count.index].id : "" ] @@ -167,7 +167,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { [ { "key" = "Name" - "value" = "${aws_eks_cluster.this.name}-${lookup( + "value" = "${aws_eks_cluster.this[0].name}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -175,7 +175,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { "propagate_at_launch" = true }, { - "key" = "kubernetes.io/cluster/${aws_eks_cluster.this.name}" + "key" = "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" "value" = "owned" "propagate_at_launch" = true }, @@ -189,8 +189,8 @@ resource "aws_autoscaling_group" "workers_launch_template" { "propagate_at_launch" = false }, { - "key" = "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}" - "value" = aws_eks_cluster.this.name + "key" = "k8s.io/cluster-autoscaler/${aws_eks_cluster.this[0].name}" + "value" = aws_eks_cluster.this[0].name "propagate_at_launch" = false }, { @@ -218,8 +218,8 @@ resource "aws_autoscaling_group" "workers_launch_template" { } resource "aws_launch_template" "workers_launch_template" { - count = local.worker_group_launch_template_count - name_prefix = "${aws_eks_cluster.this.name}-${lookup( + count = var.create_eks ? (local.worker_group_launch_template_count) : 0 + name_prefix = "${aws_eks_cluster.this[0].name}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -364,7 +364,7 @@ resource "aws_launch_template" "workers_launch_template" { tags = merge( { - "Name" = "${aws_eks_cluster.this.name}-${lookup( + "Name" = "${aws_eks_cluster.this[0].name}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -382,7 +382,7 @@ resource "aws_launch_template" "workers_launch_template" { } resource "random_pet" "workers_launch_template" { - count = local.worker_group_launch_template_count + count = var.create_eks ? local.worker_group_launch_template_count : 0 separator = "-" length = 2 @@ -401,8 +401,8 @@ resource "random_pet" "workers_launch_template" { } resource "aws_iam_instance_profile" "workers_launch_template" { - count = var.manage_worker_iam_resources ? local.worker_group_launch_template_count : 0 - name_prefix = aws_eks_cluster.this.name + count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_launch_template_count : 0 + name_prefix = aws_eks_cluster.this[0].name role = lookup( var.worker_groups_launch_template[count.index], "iam_role_id", From 45a39a098fbd9999ec7649ddf48c8cf2dee06363 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Wed, 11 Dec 2019 17:08:33 +0100 Subject: [PATCH 106/962] Minor tweaks to changelog (#629) * Tweaks to changelog * Fix examples link Existing one does not work. This is a permanent link to maintain the upgrade hints path --- CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e40caa423a..8a6efdf74b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,9 @@ project adheres to [Semantic Versioning](http://semver.org/). - Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) - Added flag `create_eks` to conditionally create resources (by @syst0m / @tbeijen) - Support for AWS EKS Managed Node Groups. (by @wmorgan6796) -- Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) -- Added a if check on `aws-auth` configmap when map_roles is empty. -- **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. Read the [docs](docs/upgrading-to-aws-auth-kubernetes-provider.md) for more info (by @sdehaes) +- Updated `instance_profile_names` and `instance_profile_arns` outputs to also consider launch template as well as asg (by @ankitwal) +- Added a if check on `aws-auth` configmap when `map_roles` is empty (by @shanmugakarna) +- **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. See Important notes below for upgrade notes (by @sdehaes) - Removed no longer used variable `write_aws_auth_config` (by @tbeijen) - Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) - Exit with error code when `aws-auth` configmap is unable to be updated (by @knittingdev) @@ -28,7 +28,7 @@ project adheres to [Semantic Versioning](http://semver.org/). The way the `aws-auth` configmap in the `kube-system` namespaces is managed has been changed. Before this was managed via kubectl using a null resources. This was changed to be managed by the terraform Kubernetes provider. To upgrade you have to add the kubernetes provider to the place you are calling the module. You can see examples in -the [examples](../examples) folder. Then you should import the configmap into Terraform: +the [examples](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/93636625740c63fd89ad8bc60ad180761288c54d/examples) folder. Then you should import the configmap into Terraform: ``` terraform import module.cluster1.kubernetes_config_map.aws_auth[0] kube-system/aws-auth @@ -43,8 +43,8 @@ You could also delete the aws-auth config map before doing an apply but this mea - **Breaking:** Allow for specifying a custom AMI for the worker nodes. (by @bmcstdio) - Added support for Windows workers AMIs (by @hodduc) - Allow for replacing the full userdata text with a `userdata_template_file` template and `userdata_template_extra_args` in `worker_groups` (by @snstanton) -- **Breaking:** The `kubectl` configuration file can now be fully-specified using `config_output_path`. Previously it was assumed that `config_output_path` referred to a directory and always ended with a forward slash. This is a breaking change if `config_output_path` does **not** end with a forward slash (which was advised against by the documentation). -- Changed logic for setting default ebs_optimized to only require maintaining a list of instance types that don't support it (by @jeffmhastings) +- **Breaking:** The `kubectl` configuration file can now be fully-specified using `config_output_path`. Previously it was assumed that `config_output_path` referred to a directory and always ended with a forward slash. This is a breaking change if `config_output_path` does **not** end with a forward slash (which was advised against by the documentation). (by @joshuaspence) +- Changed logic for setting default `ebs_optimized` to only require maintaining a list of instance types that don't support it (by @jeffmhastings) - Bumped minimum terraform version to 0.12.2 to prevent an error on yamlencode function (by @toadjaune) - Access conditional resource using join function in combination with splat syntax (by @miguelaferreira) From 7824e8b2637f86bc6c94bfa7b59ca304242cb7ce Mon Sep 17 00:00:00 2001 From: Max Williams Date: Wed, 11 Dec 2019 17:23:48 +0100 Subject: [PATCH 107/962] Creating 7.0.1 release (#630) --- CHANGELOG.md | 11 +++++++---- version | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a6efdf74b..3e8b806470 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,17 +9,13 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...HEAD)] - 2019-??-??] -- Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) - Added flag `create_eks` to conditionally create resources (by @syst0m / @tbeijen) - Support for AWS EKS Managed Node Groups. (by @wmorgan6796) -- Updated `instance_profile_names` and `instance_profile_arns` outputs to also consider launch template as well as asg (by @ankitwal) - Added a if check on `aws-auth` configmap when `map_roles` is empty (by @shanmugakarna) - **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. See Important notes below for upgrade notes (by @sdehaes) - Removed no longer used variable `write_aws_auth_config` (by @tbeijen) -- Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) - Exit with error code when `aws-auth` configmap is unable to be updated (by @knittingdev) - Fix deprecated interpolation-only expression (by @angelabad) -- Fix broken terraform plan/apply on a cluster < 1.14 (by @hodduc) - Updated required version of AWS Provider to >= v2.38.0 for Managed Node Groups (by @wmorgan6796) - Updated minimum version of Terraform to avoid a bug (by @dpiddockcmp) @@ -38,6 +34,13 @@ You could also delete the aws-auth config map before doing an apply but this mea # History +## [[v7.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.1...v7.0.0)] - 2019-12-11] + +- Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) +- Updated `instance_profile_names` and `instance_profile_arns` outputs to also consider launch template as well as asg (by @ankitwal) +- Fix broken terraform plan/apply on a cluster < 1.14 (by @hodduc) +- Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) + ## [[v7.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v7.0.0)] - 2019-10-30] - **Breaking:** Allow for specifying a custom AMI for the worker nodes. (by @bmcstdio) diff --git a/version b/version index d18077c214..f8ba35d676 100644 --- a/version +++ b/version @@ -1 +1 @@ -v7.0.0 +v7.0.1 From 11d8ee8631ec2bb98d85295d814a4dc738026704 Mon Sep 17 00:00:00 2001 From: Jeff Hastings Date: Wed, 11 Dec 2019 11:50:09 -0500 Subject: [PATCH 108/962] Fix idempotency with node group remote_access block (#625) * add kubernetes provider configuration for managed_node_groups example * use dynamic block for remote_access * update changelog --- CHANGELOG.md | 1 + examples/managed_node_groups/main.tf | 16 ++++++++++++++++ node_groups.tf | 17 +++++++++++++---- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e8b806470..38f7186f5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fix deprecated interpolation-only expression (by @angelabad) - Updated required version of AWS Provider to >= v2.38.0 for Managed Node Groups (by @wmorgan6796) - Updated minimum version of Terraform to avoid a bug (by @dpiddockcmp) +- Fix idempotency issues for node groups with no remote_access configuration (by @jeffmhastings) #### Important notes diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index 00b93aa9ff..4dd84e7a87 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -23,6 +23,22 @@ provider "template" { version = "~> 2.1" } +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false + version = "~> 1.10" +} + data "aws_availability_zones" "available" { } diff --git a/node_groups.tf b/node_groups.tf index c604da6e0c..eb2f4c310b 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -87,10 +87,19 @@ resource "aws_eks_node_group" "workers" { labels = lookup(each.value, "node_group_k8s_labels", null) release_version = lookup(each.value, "ami_release_version", null) - # This sometimes breaks idempotency as described in https://github.com/terraform-providers/terraform-provider-aws/issues/11063 - remote_access { - ec2_ssh_key = lookup(each.value, "key_name", "") != "" ? each.value["key_name"] : null - source_security_group_ids = lookup(each.value, "key_name", "") != "" ? lookup(each.value, "source_security_group_ids", []) : null + dynamic "remote_access" { + for_each = [ + for node_group in [each.value] : { + ec2_ssh_key = node_group["key_name"] + source_security_group_ids = lookup(node_group, "source_security_group_ids", []) + } + if lookup(node_group, "key_name", "") != "" + ] + + content { + ec2_ssh_key = remote_access.value["ec2_ssh_key"] + source_security_group_ids = remote_access.value["source_security_group_ids"] + } } version = aws_eks_cluster.this[0].version From 77f947ec91f6255e007d6384492fb8e690f12c92 Mon Sep 17 00:00:00 2001 From: chewvader Date: Thu, 12 Dec 2019 08:44:45 -0800 Subject: [PATCH 109/962] Cluster oidc issuer url string (#626) * cluster_oidc_issuer_url to string change cluster_oidc_issuer_url output to return string * Update CHANGELOG.md adding entry to changelog * Update outputs.tf Co-Authored-By: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> --- CHANGELOG.md | 1 + outputs.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38f7186f5b..204c7338fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fix deprecated interpolation-only expression (by @angelabad) - Updated required version of AWS Provider to >= v2.38.0 for Managed Node Groups (by @wmorgan6796) - Updated minimum version of Terraform to avoid a bug (by @dpiddockcmp) +- Fix cluster_oidc_issuer_url output from list to string (by @chewvader) - Fix idempotency issues for node groups with no remote_access configuration (by @jeffmhastings) #### Important notes diff --git a/outputs.tf b/outputs.tf index 2478dc2acb..88ab2017f4 100644 --- a/outputs.tf +++ b/outputs.tf @@ -45,7 +45,7 @@ output "cluster_iam_role_arn" { output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster OIDC Issuer" - value = concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""])[0] + value = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0] } output "cloudwatch_log_group_name" { From 583c32d2867185d3bd7154b5c5e2b9186eda0e93 Mon Sep 17 00:00:00 2001 From: Alaa Qutaish Date: Thu, 19 Dec 2019 16:38:12 +0100 Subject: [PATCH 110/962] Add IRSA support (#632) --- CHANGELOG.md | 1 + README.md | 2 ++ irsa.tf | 15 +++++++++++++++ variables.tf | 12 ++++++++++++ 4 files changed, 30 insertions(+) create mode 100644 irsa.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index 204c7338fb..e602c0fb4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Updated minimum version of Terraform to avoid a bug (by @dpiddockcmp) - Fix cluster_oidc_issuer_url output from list to string (by @chewvader) - Fix idempotency issues for node groups with no remote_access configuration (by @jeffmhastings) +- Added support to create IAM OpenID Connect Identity Provider to enable EKS Identity Roles for Service Accounts (IRSA). (by @alaa) #### Important notes diff --git a/README.md b/README.md index 20d2501f27..c01fa28728 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.14"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | string | `"./"` | no | | create\_eks | Controls if EKS resources should be created (it affects almost all resources) | bool | `"true"` | no | +| eks\_oidc\_root\_ca\_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | string | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | +| enable\_irsa | Whether to create OpenID Connect Provider for EKS to enable IRSA | bool | `"false"` | no | | iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | | kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | | kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | string | `"aws-iam-authenticator"` | no | diff --git a/irsa.tf b/irsa.tf new file mode 100644 index 0000000000..08096d8f08 --- /dev/null +++ b/irsa.tf @@ -0,0 +1,15 @@ +# Enable IAM Roles for EKS Service-Accounts (IRSA). + +# The Root CA Thumbprint for an OpenID Connect Identity Provider is currently +# Being passed as a default value which is the same for all regions and +# Is valid until (Jun 28 17:39:16 2034 GMT). +# https://crt.sh/?q=9E99A48A9960B14926BB7F3B02E22DA2B0AB7280 +# https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc_verify-thumbprint.html +# https://github.com/terraform-providers/terraform-provider-aws/issues/10104 + +resource "aws_iam_openid_connect_provider" "oidc_provider" { + count = var.enable_irsa ? 1 : 0 + client_id_list = ["sts.amazonaws.com"] + thumbprint_list = [var.eks_oidc_root_ca_thumbprint] + url = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0] +} diff --git a/variables.tf b/variables.tf index 366c9267f6..528441bc63 100644 --- a/variables.tf +++ b/variables.tf @@ -299,3 +299,15 @@ variable "node_groups" { type = any default = [] } + +variable "enable_irsa" { + description = "Whether to create OpenID Connect Provider for EKS to enable IRSA" + type = bool + default = false +} + +variable "eks_oidc_root_ca_thumbprint" { + type = string + description = "Thumbprint of Root CA for EKS OIDC, Valid until 2037" + default = "9e99a48a9960b14926bb7f3b02e22da2b0ab7280" +} From b7ffc1b591173ec7e279e2fc7597f651cde0c4c7 Mon Sep 17 00:00:00 2001 From: Ryan Ooi Date: Fri, 20 Dec 2019 23:45:01 +0800 Subject: [PATCH 111/962] Control plane security group always whitelist worker security group and revert #186 (#631) * Updates and revert #186 * update readme * update changelog * update changelog --- CHANGELOG.md | 7 +++++++ README.md | 2 -- cluster.tf | 6 +++--- local.tf | 4 ++-- variables.tf | 12 ------------ workers.tf | 12 ++++++------ 6 files changed, 18 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e602c0fb4c..bf7ce24a47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fix cluster_oidc_issuer_url output from list to string (by @chewvader) - Fix idempotency issues for node groups with no remote_access configuration (by @jeffmhastings) - Added support to create IAM OpenID Connect Identity Provider to enable EKS Identity Roles for Service Accounts (IRSA). (by @alaa) +- **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) #### Important notes @@ -35,6 +36,12 @@ terraform import module.cluster1.kubernetes_config_map.aws_auth[0] kube-system/a You could also delete the aws-auth config map before doing an apply but this means you need to the apply with the **same user/role that created the cluster**. +For security group whitelisting change. After upgrade, have to remove `cluster_create_security_group` and `worker_create_security_group` variable. If you have whitelist worker security group before, you will have to delete it(and apply again) or import it. + +``` +terraform import module.eks.aws_security_group_rule.cluster_https_worker_ingress _ingress_tcp_443_443_ +``` + # History ## [[v7.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.1...v7.0.0)] - 2019-12-11] diff --git a/README.md b/README.md index c01fa28728..03930be586 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a |------|-------------|:----:|:-----:|:-----:| | attach\_worker\_autoscaling\_policy | Whether to attach the module managed cluster autoscaling iam policy to the default worker IAM role. This requires `manage_worker_autoscaling_policy = true` | bool | `"true"` | no | | attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | bool | `"true"` | no | -| cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | bool | `"true"` | no | | cluster\_create\_timeout | Timeout value when creating the EKS cluster. | string | `"15m"` | no | | cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | string | `"15m"` | no | | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | list(string) | `[]` | no | @@ -193,7 +192,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"602401143452"` | no | | worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"801119661308"` | no | | worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | bool | `"false"` | no | -| worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | bool | `"true"` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | | worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | string | `""` | no | diff --git a/cluster.tf b/cluster.tf index b7ced5c144..d0e2fd8751 100644 --- a/cluster.tf +++ b/cluster.tf @@ -34,7 +34,7 @@ resource "aws_eks_cluster" "this" { } resource "aws_security_group" "cluster" { - count = var.cluster_create_security_group && var.create_eks ? 1 : 0 + count = var.cluster_security_group_id == "" && var.create_eks ? 1 : 0 name_prefix = var.cluster_name description = "EKS cluster security group." vpc_id = var.vpc_id @@ -47,7 +47,7 @@ resource "aws_security_group" "cluster" { } resource "aws_security_group_rule" "cluster_egress_internet" { - count = var.cluster_create_security_group && var.create_eks ? 1 : 0 + count = var.cluster_security_group_id == "" && var.create_eks ? 1 : 0 description = "Allow cluster egress access to the Internet." protocol = "-1" security_group_id = local.cluster_security_group_id @@ -58,7 +58,7 @@ resource "aws_security_group_rule" "cluster_egress_internet" { } resource "aws_security_group_rule" "cluster_https_worker_ingress" { - count = var.cluster_create_security_group && var.create_eks ? 1 : 0 + count = var.create_eks ? 1 : 0 description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" security_group_id = local.cluster_security_group_id diff --git a/local.tf b/local.tf index ba0bb72f2d..1a604dfa67 100644 --- a/local.tf +++ b/local.tf @@ -8,10 +8,10 @@ locals { ) ] - cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id + cluster_security_group_id = var.cluster_security_group_id == "" ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) - worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id + worker_security_group_id = var.worker_security_group_id == "" ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name diff --git a/variables.tf b/variables.tf index 528441bc63..f04b493f03 100644 --- a/variables.tf +++ b/variables.tf @@ -204,18 +204,6 @@ variable "local_exec_interpreter" { default = ["/bin/sh", "-c"] } -variable "cluster_create_security_group" { - description = "Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`." - type = bool - default = true -} - -variable "worker_create_security_group" { - description = "Whether to create a security group for the workers or attach the workers to `worker_security_group_id`." - type = bool - default = true -} - variable "worker_create_initial_lifecycle_hooks" { description = "Whether to create initial lifecycle hooks provided in worker groups." type = bool diff --git a/workers.tf b/workers.tf index ce0f1e2277..b6e6f69f9e 100644 --- a/workers.tf +++ b/workers.tf @@ -243,7 +243,7 @@ resource "random_pet" "workers" { } resource "aws_security_group" "workers" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 + count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 name_prefix = aws_eks_cluster.this[0].name description = "Security group for all nodes in the cluster." vpc_id = var.vpc_id @@ -257,7 +257,7 @@ resource "aws_security_group" "workers" { } resource "aws_security_group_rule" "workers_egress_internet" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 + count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 description = "Allow nodes all egress to the Internet." protocol = "-1" security_group_id = local.worker_security_group_id @@ -268,7 +268,7 @@ resource "aws_security_group_rule" "workers_egress_internet" { } resource "aws_security_group_rule" "workers_ingress_self" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 + count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 description = "Allow node to communicate with each other." protocol = "-1" security_group_id = local.worker_security_group_id @@ -279,7 +279,7 @@ resource "aws_security_group_rule" "workers_ingress_self" { } resource "aws_security_group_rule" "workers_ingress_cluster" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 + count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 description = "Allow workers pods to receive communication from the cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id @@ -290,7 +290,7 @@ resource "aws_security_group_rule" "workers_ingress_cluster" { } resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { - count = var.worker_create_security_group && var.create_eks ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 + count = var.worker_security_group_id == "" && var.create_eks ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 description = "Allow workers Kubelets to receive communication from the cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id @@ -301,7 +301,7 @@ resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { } resource "aws_security_group_rule" "workers_ingress_cluster_https" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 + count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id From 7c8bcc967b0c25b0f6881082625687349ba370c2 Mon Sep 17 00:00:00 2001 From: Mukesh Gupta Date: Fri, 20 Dec 2019 21:20:29 +0530 Subject: [PATCH 112/962] Adding node group iam role arns to outputs (#641) --- CHANGELOG.md | 1 + README.md | 1 + outputs.tf | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf7ce24a47..e1ade17516 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fix cluster_oidc_issuer_url output from list to string (by @chewvader) - Fix idempotency issues for node groups with no remote_access configuration (by @jeffmhastings) - Added support to create IAM OpenID Connect Identity Provider to enable EKS Identity Roles for Service Accounts (IRSA). (by @alaa) +- Adding node group iam role arns to outputs. (by @mukgupta) - **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) #### Important notes diff --git a/README.md b/README.md index 03930be586..23ccb7a75f 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | +| node\_groups\_iam\_role\_arns | IAM role ARNs for EKS node groups | | worker\_autoscaling\_policy\_arn | ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | | worker\_autoscaling\_policy\_name | Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | | worker\_iam\_instance\_profile\_arns | default IAM instance profile ARN for EKS worker groups | diff --git a/outputs.tf b/outputs.tf index 88ab2017f4..213f2caa40 100644 --- a/outputs.tf +++ b/outputs.tf @@ -157,3 +157,11 @@ output "worker_autoscaling_policy_arn" { description = "ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true`" value = concat(aws_iam_policy.worker_autoscaling[*].arn, [""])[0] } + +output "node_groups_iam_role_arns" { + description = "IAM role ARNs for EKS node groups" + value = { + for node_group in aws_eks_node_group.workers : + node_group.node_group_name => node_group.node_role_arn + } +} From bad9604882983219d090ea19282c12afa47ee93c Mon Sep 17 00:00:00 2001 From: Will Bertelsen Date: Fri, 20 Dec 2019 08:30:40 -0800 Subject: [PATCH 113/962] Fix aws-auth config map for managed node groups (#627) * Fix aws-auth config map for managed node groups This change adds the IAM role used for each managed node group to the aws-auth config map. This fixes an issue where managed nodes could not access the EKS kubernetes API server. * update changelog * fix format * add comment Co-authored-by: Max Williams --- CHANGELOG.md | 1 + aws_auth.tf | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1ade17516..a5864e6b6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Updated minimum version of Terraform to avoid a bug (by @dpiddockcmp) - Fix cluster_oidc_issuer_url output from list to string (by @chewvader) - Fix idempotency issues for node groups with no remote_access configuration (by @jeffmhastings) +- Fix aws-auth config map for managed node groups (by @wbertelsen) - Added support to create IAM OpenID Connect Identity Provider to enable EKS Identity Roles for Service Accounts (IRSA). (by @alaa) - Adding node group iam role arns to outputs. (by @mukgupta) - **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) diff --git a/aws_auth.tf b/aws_auth.tf index aabf6fdb94..f72654db9e 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -42,6 +42,16 @@ data "template_file" "worker_role_arns" { } } +data "template_file" "node_group_arns" { + count = var.create_eks ? local.worker_group_managed_node_group_count : 0 + template = file("${path.module}/templates/worker-role.tpl") + + vars = { + worker_role_arn = lookup(var.node_groups[count.index], "iam_role_arn", aws_iam_role.node_groups[0].arn) + platform = "linux" # Hardcoded because the EKS API currently only supports linux for managed node groups + } +} + resource "kubernetes_config_map" "aws_auth" { count = var.create_eks && var.manage_aws_auth ? 1 : 0 @@ -51,11 +61,12 @@ resource "kubernetes_config_map" "aws_auth" { } data = { - mapRoles = < Date: Fri, 3 Jan 2020 05:25:09 -0800 Subject: [PATCH 114/962] Add the OIDC provider ARN to outputs (#653) --- CHANGELOG.md | 1 + README.md | 1 + outputs.tf | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5864e6b6d..e19b8c6a6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fix aws-auth config map for managed node groups (by @wbertelsen) - Added support to create IAM OpenID Connect Identity Provider to enable EKS Identity Roles for Service Accounts (IRSA). (by @alaa) - Adding node group iam role arns to outputs. (by @mukgupta) +- Added the OIDC Provider ARN to outputs. (by @eytanhanig) - **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) #### Important notes diff --git a/README.md b/README.md index 23ccb7a75f..059e13294a 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | | node\_groups\_iam\_role\_arns | IAM role ARNs for EKS node groups | +| oidc\_provider\_arn | The ARN of the OIDC Provider if `enable_irsa = true`. | | worker\_autoscaling\_policy\_arn | ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | | worker\_autoscaling\_policy\_name | Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | | worker\_iam\_instance\_profile\_arns | default IAM instance profile ARN for EKS worker groups | diff --git a/outputs.tf b/outputs.tf index 213f2caa40..34e0064779 100644 --- a/outputs.tf +++ b/outputs.tf @@ -63,6 +63,11 @@ output "kubeconfig_filename" { value = concat(local_file.kubeconfig.*.filename, [""])[0] } +output "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`." + value = var.enable_irsa ? aws_iam_openid_connect_provider.oidc_provider[0].arn : null +} + output "workers_asg_arns" { description = "IDs of the autoscaling groups containing workers." value = concat( From c17d17e2c251ce9f57fcab6a6822ca138f01f035 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 3 Jan 2020 23:12:41 +0100 Subject: [PATCH 115/962] add stale app (#657) --- .github/stale.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/stale.yml diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 0000000000..894f3a0efb --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,35 @@ +--- +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 90 + +# Number of days of inactivity before an stale issue is closed +daysUntilClose: 30 + +# Label to use when marking an issue as stale +staleLabel: stale + +issues: + # Comment to post when marking an issue as stale. + markComment: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. + # Comment to post when closing a stale issue. + closeComment: > + This issue has been automatically closed because it has not had recent + activity since being marked as stale. +pulls: + # Comment to post when marking a PR as stale. + markComment: > + This PR has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. + + To track this PR (even if closed), please open a corresponding issue if one + does not already exist. + # Comment to post when closing a stale PR. + closeComment: > + This PR has been automatically closed because it has not had recent + activity since being marked as stale. + + Please reopen when work resumes. From d79c8ab6f2a8f90b0ac8cf36fd6f75d08eb111ba Mon Sep 17 00:00:00 2001 From: Shaun Cutts Date: Tue, 7 Jan 2020 06:28:56 -0500 Subject: [PATCH 116/962] Wait cluster responsive (#639) * wait for cluster to respond before creating auth config map * adds changelog entry * fixup tf format * fixup kubernetes required version * fixup missing local for kubeconfig_filename * combine wait for cluster into provisioner on cluster; change status check to /healthz on endpoint * fix: make kubernetes provider version more permissive --- CHANGELOG.md | 1 + aws_auth.tf | 3 ++- cluster.tf | 5 +++++ versions.tf | 11 ++++++----- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e19b8c6a6f..c3c19fd736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...HEAD)] - 2019-??-??] +- Wait for cluster to respond to kubectl before applying auth map_config (@shaunc) - Added flag `create_eks` to conditionally create resources (by @syst0m / @tbeijen) - Support for AWS EKS Managed Node Groups. (by @wmorgan6796) - Added a if check on `aws-auth` configmap when `map_roles` is empty (by @shanmugakarna) diff --git a/aws_auth.tf b/aws_auth.tf index f72654db9e..a2a25ec134 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -53,7 +53,8 @@ data "template_file" "node_group_arns" { } resource "kubernetes_config_map" "aws_auth" { - count = var.create_eks && var.manage_aws_auth ? 1 : 0 + depends_on = [aws_eks_cluster.this] + count = var.create_eks && var.manage_aws_auth ? 1 : 0 metadata { name = "aws-auth" diff --git a/cluster.tf b/cluster.tf index d0e2fd8751..86ff69a29a 100644 --- a/cluster.tf +++ b/cluster.tf @@ -31,6 +31,11 @@ resource "aws_eks_cluster" "this" { aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy, aws_cloudwatch_log_group.this ] + provisioner "local-exec" { + command = </dev/null; do sleep 4; done + EOT + } } resource "aws_security_group" "cluster" { diff --git a/versions.tf b/versions.tf index 06a6c1cc50..e95ea3e9d3 100644 --- a/versions.tf +++ b/versions.tf @@ -2,10 +2,11 @@ terraform { required_version = ">= 0.12.9" required_providers { - aws = ">= 2.38.0" - local = ">= 1.2" - null = ">= 2.1" - template = ">= 2.1" - random = ">= 2.1" + aws = ">= 2.38.0" + local = ">= 1.2" + null = ">= 2.1" + template = ">= 2.1" + random = ">= 2.1" + kubernetes = ">= 1.6.2" } } From 11147e9af34054c4c4576aa00938a2c65198ca5f Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Thu, 9 Jan 2020 12:53:08 +0100 Subject: [PATCH 117/962] Node groups submodule (#650) * WIP Move node_groups to a submodule * Split the old node_groups file up * Start moving locals * Simplify IAM creation logic * depends_on from the TF docs * Wire in the variables * Call module from parent * Allow to customize the role name. As per workers * aws_auth ConfigMap for node_groups * Get the managed_node_groups example to plan * Get the basic example to plan too * create_eks = false works "The true and false result expressions must have consistent types. The given expressions are object and object, respectively." Well, that's useful. But apparently set(string) and set() are ok. So everything else is more complicated. Thanks. * Update Changelog * Update README * Wire in node_groups_defaults * Remove node_groups from workers_defaults_defaults * Synchronize random and node_group defaults * Error: "name_prefix" cannot be longer than 32 * Update READMEs again * Fix double destroy Was producing index errors when running destroy on an empty state. * Remove duplicate iam_role in node_group I think this logic works. Needs some testing with an externally created role. * Fix index fail if node group manually deleted * Keep aws_auth template in top module Downside: count causes issues as usual: can't use distinct() in the child module so there's a template render for every node_group even if only one role is really in use. Hopefully just output noise instead of technical issue * Hack to have node_groups depend on aws_auth etc The AWS Node Groups create or edit the aws-auth ConfigMap so that nodes can join the cluster. This breaks the kubernetes resource which cannot do a force create. Remove the race condition with explicit depend. Can't pull the IAM role out of the node_group any more. * Pull variables via the random_pet to cut logic No point having the same logic in two different places * Pass all ForceNew variables through the pet * Do a deep merge of NG labels and tags * Update README.. again * Additional managed node outputs #644 Add change from @TBeijin from PR #644 * Remove unused local * Use more for_each * Remove the change when create_eks = false * Make documentation less confusing * node_group version user configurable * Pass through raw output from aws_eks_node_groups * Merge workers defaults in the locals This simplifies the random_pet and aws_eks_node_group logic. Which was causing much consernation on the PR. * Fix typo Co-authored-by: Max Williams --- CHANGELOG.md | 2 + README.md | 5 +- aws_auth.tf | 7 +- examples/managed_node_groups/main.tf | 26 ++--- examples/managed_node_groups/outputs.tf | 4 + local.tf | 17 +-- modules/node_groups/README.md | 55 ++++++++++ modules/node_groups/locals.tf | 16 +++ modules/node_groups/node_groups.tf | 49 +++++++++ modules/node_groups/outputs.tf | 14 +++ modules/node_groups/random.tf | 21 ++++ modules/node_groups/variables.tf | 36 +++++++ node_groups.tf | 133 +++++------------------- outputs.tf | 9 +- variables.tf | 10 +- 15 files changed, 254 insertions(+), 150 deletions(-) create mode 100644 modules/node_groups/README.md create mode 100644 modules/node_groups/locals.tf create mode 100644 modules/node_groups/node_groups.tf create mode 100644 modules/node_groups/outputs.tf create mode 100644 modules/node_groups/random.tf create mode 100644 modules/node_groups/variables.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index c3c19fd736..f1e6b00180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ project adheres to [Semantic Versioning](http://semver.org/). - Adding node group iam role arns to outputs. (by @mukgupta) - Added the OIDC Provider ARN to outputs. (by @eytanhanig) - **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) +- Move `eks_node_group` resources to a submodule (by @dpiddockcmp) +- Add complex output `node_groups` (by @TBeijen) #### Important notes diff --git a/README.md b/README.md index 059e13294a..05dbed13c9 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | | map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | | map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | -| node\_groups | A list of maps defining node group configurations to be defined using AWS EKS Managed Node Groups. See workers_group_defaults for valid keys. | any | `[]` | no | +| node\_groups | Map of map of node groups to create. See `node_groups` module's documentation for more details | any | `{}` | no | +| node\_groups\_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentaton for more details | any | `{}` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `"null"` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | | tags | A map of tags to add to all resources. | map(string) | `{}` | no | @@ -218,7 +219,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | -| node\_groups\_iam\_role\_arns | IAM role ARNs for EKS node groups | +| node\_groups | Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys | | oidc\_provider\_arn | The ARN of the OIDC Provider if `enable_irsa = true`. | | worker\_autoscaling\_policy\_arn | ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | | worker\_autoscaling\_policy\_name | Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | diff --git a/aws_auth.tf b/aws_auth.tf index a2a25ec134..cce8f667c4 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -43,13 +43,10 @@ data "template_file" "worker_role_arns" { } data "template_file" "node_group_arns" { - count = var.create_eks ? local.worker_group_managed_node_group_count : 0 + count = var.create_eks ? length(module.node_groups.aws_auth_roles) : 0 template = file("${path.module}/templates/worker-role.tpl") - vars = { - worker_role_arn = lookup(var.node_groups[count.index], "iam_role_arn", aws_iam_role.node_groups[0].arn) - platform = "linux" # Hardcoded because the EKS API currently only supports linux for managed node groups - } + vars = module.node_groups.aws_auth_roles[count.index] } resource "kubernetes_config_map" "aws_auth" { diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index 4dd84e7a87..db55e7d220 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -92,27 +92,29 @@ module "eks" { vpc_id = module.vpc.vpc_id - node_groups = [ - { - name = "example" + node_groups_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + } - node_group_desired_capacity = 1 - node_group_max_capacity = 10 - node_group_min_capacity = 1 + node_groups = { + example = { + desired_capacity = 1 + max_capacity = 10 + min_capacity = 1 instance_type = "m5.large" - node_group_k8s_labels = { + k8s_labels = { Environment = "test" GithubRepo = "terraform-aws-eks" GithubOrg = "terraform-aws-modules" } - node_group_additional_tags = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" + additional_tags = { + ExtraTag = "example" } } - ] + defaults = {} + } map_roles = var.map_roles map_users = var.map_users diff --git a/examples/managed_node_groups/outputs.tf b/examples/managed_node_groups/outputs.tf index a0788aff1d..7010db294f 100644 --- a/examples/managed_node_groups/outputs.tf +++ b/examples/managed_node_groups/outputs.tf @@ -23,3 +23,7 @@ output "region" { value = var.region } +output "node_groups" { + description = "Outputs from node groups" + value = module.eks.node_groups +} diff --git a/local.tf b/local.tf index 1a604dfa67..609185816f 100644 --- a/local.tf +++ b/local.tf @@ -16,9 +16,8 @@ locals { default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name - worker_group_count = length(var.worker_groups) - worker_group_launch_template_count = length(var.worker_groups_launch_template) - worker_group_managed_node_group_count = length(var.node_groups) + worker_group_count = length(var.worker_groups) + worker_group_launch_template_count = length(var.worker_groups_launch_template) default_ami_id_linux = data.aws_ami.eks_worker.id default_ami_id_windows = data.aws_ami.eks_worker_windows.id @@ -80,15 +79,6 @@ locals { spot_allocation_strategy = "lowest-price" # Valid options are 'lowest-price' and 'capacity-optimized'. If 'lowest-price', the Auto Scaling group launches instances using the Spot pools with the lowest price, and evenly allocates your instances across the number of Spot pools. If 'capacity-optimized', the Auto Scaling group launches instances using Spot pools that are optimally chosen based on the available Spot capacity. spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price - ami_type = "AL2_x86_64" # AMI Type to use for the Managed Node Groups. Can be either: AL2_x86_64 or AL2_x86_64_GPU - ami_release_version = "" # AMI Release Version of the Managed Node Groups - source_security_group_id = [] # Source Security Group IDs to allow SSH Access to the Nodes. NOTE: IF LEFT BLANK, AND A KEY IS SPECIFIED, THE SSH PORT WILL BE OPENNED TO THE WORLD - node_group_k8s_labels = {} # Kubernetes Labels to apply to the nodes within the Managed Node Group - node_group_desired_capacity = 1 # Desired capacity of the Node Group - node_group_min_capacity = 1 # Min capacity of the Node Group (Minimum value allowed is 1) - node_group_max_capacity = 3 # Max capacity of the Node Group - node_group_iam_role_arn = "" # IAM role to use for Managed Node Groups instead of default one created by the automation - node_group_additional_tags = {} # Additional tags to be applied to the Node Groups } workers_group_defaults = merge( @@ -133,7 +123,4 @@ locals { "t2.small", "t2.xlarge" ] - - node_groups = { for node_group in var.node_groups : node_group["name"] => node_group } - } diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md new file mode 100644 index 0000000000..76278d9dc4 --- /dev/null +++ b/modules/node_groups/README.md @@ -0,0 +1,55 @@ +# eks `node_groups` submodule + +Helper submodule to create and manage resources related to `eks_node_groups`. + +## Assumptions +* Designed for use by the parent module and not directly by end users + +## Node Groups' IAM Role +The role ARN specified in `var.default_iam_role_arn` will be used by default. In a simple configuration this will be the worker role created by the parent module. + +`iam_role_arn` must be specified in either `var.node_groups_defaults` or `var.node_groups` if the default parent IAM role is not being created for whatever reason, for example if `manage_worker_iam_resources` is set to false in the parent. + +## `node_groups` and `node_groups_defaults` keys +`node_groups_defaults` is a map that can take the below keys. Values will be used if not specified in individual node groups. + +`node_groups` is a map of maps. Key of first level will be used as unique value for `for_each` resources and in the `aws_eks_node_group` name. Inner map can take the below values. + +| Name | Description | Type | If unset | +|------|-------------|:----:|:-----:| +| additional\_tags | Additional tags to apply to node group | map(string) | Only `var.tags` applied | +| ami\_release\_version | AMI version of workers | string | Provider default behavior | +| ami\_type | AMI Type. See Terraform or AWS docs | string | Provider default behavior | +| desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` | +| disk\_size | Workers' disk size | number | Provider default behavior | +| iam\_role\_arn | IAM role ARN for workers | string | `var.default_iam_role_arn` | +| instance\_type | Workers' instance type | string | `var.workers_group_defaults[instance_type]` | +| k8s\_labels | Kubernetes labels | map(string) | No labels applied | +| key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` | +| max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` | +| min\_capacity | Min number of workers | number | `var.workers_group_defaults[asg_min_size]` | +| source\_security\_group\_ids | Source security groups for remote access to workers | list(string) | If key\_name is specified: THE REMOTE ACCESS WILL BE OPENED TO THE WORLD | +| subnets | Subnets to contain workers | list(string) | `var.workers_group_defaults[subnets]` | +| version | Kubernetes version | string | Provider default behavior | + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cluster\_name | Name of parent cluster | string | n/a | yes | +| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | bool | `"true"` | no | +| default\_iam\_role\_arn | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | string | n/a | yes | +| node\_groups | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | any | `{}` | no | +| node\_groups\_defaults | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | any | n/a | yes | +| tags | A map of tags to add to all resources | map(string) | n/a | yes | +| workers\_group\_defaults | Workers group defaults from parent | any | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| aws\_auth\_roles | Roles for use in aws-auth ConfigMap | +| node\_groups | Outputs from EKS node groups. Map of maps, keyed by `var.node_groups` keys. See `aws_eks_node_group` Terraform documentation for values | + + diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf new file mode 100644 index 0000000000..43cf672ca0 --- /dev/null +++ b/modules/node_groups/locals.tf @@ -0,0 +1,16 @@ +locals { + # Merge defaults and per-group values to make code cleaner + node_groups_expanded = { for k, v in var.node_groups : k => merge( + { + desired_capacity = var.workers_group_defaults["asg_desired_capacity"] + iam_role_arn = var.default_iam_role_arn + instance_type = var.workers_group_defaults["instance_type"] + key_name = var.workers_group_defaults["key_name"] + max_capacity = var.workers_group_defaults["asg_max_size"] + min_capacity = var.workers_group_defaults["asg_min_size"] + subnets = var.workers_group_defaults["subnets"] + }, + var.node_groups_defaults, + v, + ) if var.create_eks } +} diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf new file mode 100644 index 0000000000..cdbc6d00b3 --- /dev/null +++ b/modules/node_groups/node_groups.tf @@ -0,0 +1,49 @@ +resource "aws_eks_node_group" "workers" { + for_each = local.node_groups_expanded + + node_group_name = join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id]) + + cluster_name = var.cluster_name + node_role_arn = each.value["iam_role_arn"] + subnet_ids = each.value["subnets"] + + scaling_config { + desired_size = each.value["desired_capacity"] + max_size = each.value["max_capacity"] + min_size = each.value["min_capacity"] + } + + ami_type = lookup(each.value, "ami_type", null) + disk_size = lookup(each.value, "disk_size", null) + instance_types = [each.value["instance_type"]] + release_version = lookup(each.value, "ami_release_version", null) + + dynamic "remote_access" { + for_each = each.value["key_name"] != "" ? [{ + ec2_ssh_key = each.value["key_name"] + source_security_group_ids = lookup(each.value, "source_security_group_ids", []) + }] : [] + + content { + ec2_ssh_key = remote_access.value["ec2_ssh_key"] + source_security_group_ids = remote_access.value["source_security_group_ids"] + } + } + + version = lookup(each.value, "version", null) + + labels = merge( + lookup(var.node_groups_defaults, "k8s_labels", {}), + lookup(var.node_groups[each.key], "k8s_labels", {}) + ) + + tags = merge( + var.tags, + lookup(var.node_groups_defaults, "additional_tags", {}), + lookup(var.node_groups[each.key], "additional_tags", {}), + ) + + lifecycle { + create_before_destroy = true + } +} diff --git a/modules/node_groups/outputs.tf b/modules/node_groups/outputs.tf new file mode 100644 index 0000000000..ad148ea514 --- /dev/null +++ b/modules/node_groups/outputs.tf @@ -0,0 +1,14 @@ +output "node_groups" { + description = "Outputs from EKS node groups. Map of maps, keyed by `var.node_groups` keys. See `aws_eks_node_group` Terraform documentation for values" + value = aws_eks_node_group.workers +} + +output "aws_auth_roles" { + description = "Roles for use in aws-auth ConfigMap" + value = [ + for k, v in local.node_groups_expanded : { + worker_role_arn = lookup(v, "iam_role_arn", var.default_iam_role_arn) + platform = "linux" + } + ] +} diff --git a/modules/node_groups/random.tf b/modules/node_groups/random.tf new file mode 100644 index 0000000000..14e7ba2bce --- /dev/null +++ b/modules/node_groups/random.tf @@ -0,0 +1,21 @@ +resource "random_pet" "node_groups" { + for_each = local.node_groups_expanded + + separator = "-" + length = 2 + + keepers = { + ami_type = lookup(each.value, "ami_type", null) + disk_size = lookup(each.value, "disk_size", null) + instance_type = each.value["instance_type"] + iam_role_arn = each.value["iam_role_arn"] + + key_name = each.value["key_name"] + + source_security_group_ids = join("|", compact( + lookup(each.value, "source_security_group_ids", []) + )) + subnet_ids = join("|", each.value["subnets"]) + node_group_name = join("-", [var.cluster_name, each.key]) + } +} diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf new file mode 100644 index 0000000000..c0eaa23d1e --- /dev/null +++ b/modules/node_groups/variables.tf @@ -0,0 +1,36 @@ +variable "create_eks" { + description = "Controls if EKS resources should be created (it affects almost all resources)" + type = bool + default = true +} + +variable "cluster_name" { + description = "Name of parent cluster" + type = string +} + +variable "default_iam_role_arn" { + description = "ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults`" + type = string +} + +variable "workers_group_defaults" { + description = "Workers group defaults from parent" + type = any +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) +} + +variable "node_groups_defaults" { + description = "map of maps of node groups to create. See \"`node_groups` and `node_groups_defaults` keys\" section in README.md for more details" + type = any +} + +variable "node_groups" { + description = "Map of maps of `eks_node_groups` to create. See \"`node_groups` and `node_groups_defaults` keys\" section in README.md for more details" + type = any + default = {} +} diff --git a/node_groups.tf b/node_groups.tf index eb2f4c310b..6c7b438cfb 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -1,112 +1,29 @@ -resource "aws_iam_role" "node_groups" { - count = var.create_eks && local.worker_group_managed_node_group_count > 0 ? 1 : 0 - name = "${var.workers_role_name != "" ? var.workers_role_name : aws_eks_cluster.this[0].name}-managed-node-groups" - assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json - permissions_boundary = var.permissions_boundary - path = var.iam_path - force_detach_policies = true - tags = var.tags -} - -resource "aws_iam_role_policy_attachment" "node_groups_AmazonEKSWorkerNodePolicy" { - count = var.create_eks && local.worker_group_managed_node_group_count > 0 ? 1 : 0 - policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" - role = aws_iam_role.node_groups[0].name -} - -resource "aws_iam_role_policy_attachment" "node_groups_AmazonEKS_CNI_Policy" { - count = var.create_eks && local.worker_group_managed_node_group_count > 0 ? 1 : 0 - policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" - role = aws_iam_role.node_groups[0].name -} - -resource "aws_iam_role_policy_attachment" "node_groups_AmazonEC2ContainerRegistryReadOnly" { - count = var.create_eks && local.worker_group_managed_node_group_count > 0 ? 1 : 0 - policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" - role = aws_iam_role.node_groups[0].name -} - -resource "aws_iam_role_policy_attachment" "node_groups_additional_policies" { - for_each = var.create_eks && local.worker_group_managed_node_group_count > 0 ? toset(var.workers_additional_policies) : [] - - role = aws_iam_role.node_groups[0].name - policy_arn = each.key -} - -resource "aws_iam_role_policy_attachment" "node_groups_autoscaling" { - count = var.create_eks && var.manage_worker_autoscaling_policy && var.attach_worker_autoscaling_policy && local.worker_group_managed_node_group_count > 0 ? 1 : 0 - policy_arn = aws_iam_policy.node_groups_autoscaling[0].arn - role = aws_iam_role.node_groups[0].name -} - -resource "aws_iam_policy" "node_groups_autoscaling" { - count = var.create_eks && var.manage_worker_autoscaling_policy && local.worker_group_managed_node_group_count > 0 ? 1 : 0 - name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this[0].name}" - description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this[0].name}" - policy = data.aws_iam_policy_document.worker_autoscaling[0].json - path = var.iam_path -} - -resource "random_pet" "node_groups" { - for_each = var.create_eks ? local.node_groups : {} - - separator = "-" - length = 2 - - keepers = { - instance_type = lookup(each.value, "instance_type", local.workers_group_defaults["instance_type"]) - - ec2_ssh_key = lookup(each.value, "key_name", local.workers_group_defaults["key_name"]) - - source_security_group_ids = join("-", compact( - lookup(each.value, "source_security_group_ids", local.workers_group_defaults["source_security_group_id"] - ))) - - node_group_name = join("-", [var.cluster_name, each.value["name"]]) +# Hack to ensure ordering of resource creation. Do not create node_groups +# before other resources are ready. Removes race conditions +data "null_data_source" "node_groups" { + count = var.create_eks ? 1 : 0 + + inputs = { + cluster_name = var.cluster_name + + # Ensure these resources are created before "unlocking" the data source. + # `depends_on` causes a refresh on every run so is useless here. + # [Re]creating or removing these resources will trigger recreation of Node Group resources + aws_auth = coalescelist(kubernetes_config_map.aws_auth[*].id, [""])[0] + role_NodePolicy = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy[*].id, [""])[0] + role_CNI_Policy = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy[*].id, [""])[0] + role_Container = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly[*].id, [""])[0] + role_autoscaling = coalescelist(aws_iam_role_policy_attachment.workers_autoscaling[*].id, [""])[0] } } -resource "aws_eks_node_group" "workers" { - for_each = var.create_eks ? local.node_groups : {} - - node_group_name = join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id]) - - cluster_name = var.cluster_name - node_role_arn = lookup(each.value, "iam_role_arn", aws_iam_role.node_groups[0].arn) - subnet_ids = lookup(each.value, "subnets", local.workers_group_defaults["subnets"]) - - scaling_config { - desired_size = lookup(each.value, "node_group_desired_capacity", local.workers_group_defaults["asg_desired_capacity"]) - max_size = lookup(each.value, "node_group_max_capacity", local.workers_group_defaults["asg_max_size"]) - min_size = lookup(each.value, "node_group_min_capacity", local.workers_group_defaults["asg_min_size"]) - } - - ami_type = lookup(each.value, "ami_type", null) - disk_size = lookup(each.value, "root_volume_size", null) - instance_types = [lookup(each.value, "instance_type", null)] - labels = lookup(each.value, "node_group_k8s_labels", null) - release_version = lookup(each.value, "ami_release_version", null) - - dynamic "remote_access" { - for_each = [ - for node_group in [each.value] : { - ec2_ssh_key = node_group["key_name"] - source_security_group_ids = lookup(node_group, "source_security_group_ids", []) - } - if lookup(node_group, "key_name", "") != "" - ] - - content { - ec2_ssh_key = remote_access.value["ec2_ssh_key"] - source_security_group_ids = remote_access.value["source_security_group_ids"] - } - } - - version = aws_eks_cluster.this[0].version - - tags = lookup(each.value, "node_group_additional_tags", null) - - lifecycle { - create_before_destroy = true - } +module "node_groups" { + source = "./modules/node_groups" + create_eks = var.create_eks + cluster_name = coalescelist(data.null_data_source.node_groups[*].outputs["cluster_name"], [""])[0] + default_iam_role_arn = coalescelist(aws_iam_role.workers[*].arn, [""])[0] + workers_group_defaults = local.workers_group_defaults + tags = var.tags + node_groups_defaults = var.node_groups_defaults + node_groups = var.node_groups } diff --git a/outputs.tf b/outputs.tf index 34e0064779..e72b29457e 100644 --- a/outputs.tf +++ b/outputs.tf @@ -163,10 +163,7 @@ output "worker_autoscaling_policy_arn" { value = concat(aws_iam_policy.worker_autoscaling[*].arn, [""])[0] } -output "node_groups_iam_role_arns" { - description = "IAM role ARNs for EKS node groups" - value = { - for node_group in aws_eks_node_group.workers : - node_group.node_group_name => node_group.node_role_arn - } +output "node_groups" { + description = "Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys" + value = module.node_groups.node_groups } diff --git a/variables.tf b/variables.tf index f04b493f03..2b64a9a28a 100644 --- a/variables.tf +++ b/variables.tf @@ -282,10 +282,16 @@ variable "create_eks" { default = true } +variable "node_groups_defaults" { + description = "Map of values to be applied to all node groups. See `node_groups` module's documentaton for more details" + type = any + default = {} +} + variable "node_groups" { - description = "A list of maps defining node group configurations to be defined using AWS EKS Managed Node Groups. See workers_group_defaults for valid keys." + description = "Map of map of node groups to create. See `node_groups` module's documentation for more details" type = any - default = [] + default = {} } variable "enable_irsa" { From a9db852d44395391c2e110ceeb5aadbf009be39c Mon Sep 17 00:00:00 2001 From: Max Williams Date: Thu, 9 Jan 2020 14:10:47 +0100 Subject: [PATCH 118/962] Release 8.0.0 (#662) * Release 8.0.0 * Update changelog * remove 'defauls' node group * Make curl silent --- CHANGELOG.md | 12 +++++++++--- cluster.tf | 2 +- examples/managed_node_groups/main.tf | 1 - version | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1e6b00180..34c1538c35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,20 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...HEAD)] - 2019-??-??] +## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...HEAD)] - 2019-12-11] +- Write your awesome change here (by @you) + +# History + +## [[v8.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v7.0.1)] - 2019-12-11] + +- **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) +- **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. See Important notes below for upgrade notes (by @sdehaes) - Wait for cluster to respond to kubectl before applying auth map_config (@shaunc) - Added flag `create_eks` to conditionally create resources (by @syst0m / @tbeijen) - Support for AWS EKS Managed Node Groups. (by @wmorgan6796) - Added a if check on `aws-auth` configmap when `map_roles` is empty (by @shanmugakarna) -- **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. See Important notes below for upgrade notes (by @sdehaes) - Removed no longer used variable `write_aws_auth_config` (by @tbeijen) - Exit with error code when `aws-auth` configmap is unable to be updated (by @knittingdev) - Fix deprecated interpolation-only expression (by @angelabad) @@ -25,7 +32,6 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added support to create IAM OpenID Connect Identity Provider to enable EKS Identity Roles for Service Accounts (IRSA). (by @alaa) - Adding node group iam role arns to outputs. (by @mukgupta) - Added the OIDC Provider ARN to outputs. (by @eytanhanig) -- **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) - Move `eks_node_group` resources to a submodule (by @dpiddockcmp) - Add complex output `node_groups` (by @TBeijen) diff --git a/cluster.tf b/cluster.tf index 86ff69a29a..764c737ff0 100644 --- a/cluster.tf +++ b/cluster.tf @@ -33,7 +33,7 @@ resource "aws_eks_cluster" "this" { ] provisioner "local-exec" { command = </dev/null; do sleep 4; done + until curl -k -s ${aws_eks_cluster.this[0].endpoint}/healthz >/dev/null; do sleep 4; done EOT } } diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index db55e7d220..c31abb36b5 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -113,7 +113,6 @@ module "eks" { ExtraTag = "example" } } - defaults = {} } map_roles = var.map_roles diff --git a/version b/version index f8ba35d676..5f4f91fb4f 100644 --- a/version +++ b/version @@ -1 +1 @@ -v7.0.1 +v8.0.0 From 82aefb20f5dc1e4d195a3d19d6c091e1ed405932 Mon Sep 17 00:00:00 2001 From: Siddarth Prakash <1428486+sidprak@users.noreply.github.com> Date: Thu, 9 Jan 2020 18:53:33 -0500 Subject: [PATCH 119/962] Add public access endpoint CIDRs option (terraform-aws-eks#647) (#673) * Add public access endpoint CIDRs option (terraform-aws-eks#647) * Update required provider version to 2.44.0 * Fix formatting in docs --- CHANGELOG.md | 1 + README.md | 1 + cluster.tf | 1 + variables.tf | 6 ++++++ versions.tf | 2 +- 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34c1538c35..63cf796f17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...HEAD)] - 2019-12-11] - Write your awesome change here (by @you) +- Add support for restricting access to the public API endpoint (@sidprak) # History diff --git a/README.md b/README.md index 05dbed13c9..b22f5739e5 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | list(string) | `[]` | no | | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | bool | `"false"` | no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | bool | `"true"` | no | +| cluster\_endpoint\_public\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | list(string) | `[ "0.0.0.0/0" ]` | no | | cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. | string | `""` | no | | cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | string | `""` | no | | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | diff --git a/cluster.tf b/cluster.tf index 764c737ff0..877ddda5c7 100644 --- a/cluster.tf +++ b/cluster.tf @@ -19,6 +19,7 @@ resource "aws_eks_cluster" "this" { subnet_ids = var.subnets endpoint_private_access = var.cluster_endpoint_private_access endpoint_public_access = var.cluster_endpoint_public_access + public_access_cidrs = var.cluster_endpoint_public_access_cidrs } timeouts { diff --git a/variables.tf b/variables.tf index 2b64a9a28a..92b906eadc 100644 --- a/variables.tf +++ b/variables.tf @@ -234,6 +234,12 @@ variable "cluster_endpoint_public_access" { default = true } +variable "cluster_endpoint_public_access_cidrs" { + description = "List of CIDR blocks which can access the Amazon EKS public API server endpoint." + type = list(string) + default = ["0.0.0.0/0"] +} + variable "manage_cluster_iam_resources" { description = "Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified." type = bool diff --git a/versions.tf b/versions.tf index e95ea3e9d3..95fb1ef19e 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.12.9" required_providers { - aws = ">= 2.38.0" + aws = ">= 2.44.0" local = ">= 1.2" null = ">= 2.1" template = ">= 2.1" From c5f50d59692c14a2133c4d4a2f961d87e9c78e35 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 13 Jan 2020 14:39:59 +0100 Subject: [PATCH 120/962] Re-generate docs with terraform-docs 0.7.0 and bump pre-commit-terraform version (#668) * re-generate docs with terraform-docs 0.7.0 * bump pre-commit-terraform version --- .pre-commit-config.yaml | 4 +- README.md | 156 +++++++++++++++++----------------- modules/node_groups/README.md | 16 ++-- 3 files changed, 89 insertions(+), 87 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f21c517082..e73233f2ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,9 @@ +repos: - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.19.0 + rev: v1.22.0 hooks: - id: terraform_fmt - id: terraform_docs + args: [--args=--with-aggregate-type-defaults --no-escape] - id: terraform_validate - id: terraform_tflint diff --git a/README.md b/README.md index b22f5739e5..f1b2c79c1b 100644 --- a/README.md +++ b/README.md @@ -150,91 +150,91 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| attach\_worker\_autoscaling\_policy | Whether to attach the module managed cluster autoscaling iam policy to the default worker IAM role. This requires `manage_worker_autoscaling_policy = true` | bool | `"true"` | no | -| attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | bool | `"true"` | no | -| cluster\_create\_timeout | Timeout value when creating the EKS cluster. | string | `"15m"` | no | -| cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | string | `"15m"` | no | -| cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | list(string) | `[]` | no | -| cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | bool | `"false"` | no | -| cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | bool | `"true"` | no | -| cluster\_endpoint\_public\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | list(string) | `[ "0.0.0.0/0" ]` | no | -| cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. | string | `""` | no | -| cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | string | `""` | no | -| cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | -| cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | -| cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | string | `""` | no | -| cluster\_version | Kubernetes version to use for the EKS cluster. | string | `"1.14"` | no | -| config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | string | `"./"` | no | -| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | bool | `"true"` | no | -| eks\_oidc\_root\_ca\_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | string | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | -| enable\_irsa | Whether to create OpenID Connect Provider for EKS to enable IRSA | bool | `"false"` | no | -| iam\_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | -| kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | -| kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | string | `"aws-iam-authenticator"` | no | -| kubeconfig\_aws\_authenticator\_command\_args | Default arguments passed to the authenticator command. Defaults to [token -i $cluster_name]. | list(string) | `[]` | no | -| kubeconfig\_aws\_authenticator\_env\_variables | Environment variables that should be used when executing the authenticator. e.g. { AWS_PROFILE = "eks"}. | map(string) | `{}` | no | -| kubeconfig\_name | Override the default name used for items kubeconfig. | string | `""` | no | -| local\_exec\_interpreter | Command to run for local-exec resources. Must be a shell-style interpreter. If you are on Windows Git Bash is a good choice. | list(string) | `[ "/bin/sh", "-c" ]` | no | -| manage\_aws\_auth | Whether to apply the aws-auth configmap file. | string | `"true"` | no | -| manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | bool | `"true"` | no | -| manage\_worker\_autoscaling\_policy | Whether to let the module manage the cluster autoscaling iam policy. | bool | `"true"` | no | -| manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | bool | `"true"` | no | -| map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | -| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | -| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | -| node\_groups | Map of map of node groups to create. See `node_groups` module's documentation for more details | any | `{}` | no | -| node\_groups\_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentaton for more details | any | `{}` | no | -| permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `"null"` | no | +| attach_worker_autoscaling_policy | Whether to attach the module managed cluster autoscaling iam policy to the default worker IAM role. This requires `manage_worker_autoscaling_policy = true` | bool | `"true"` | no | +| attach_worker_cni_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | bool | `"true"` | no | +| cluster_create_timeout | Timeout value when creating the EKS cluster. | string | `"15m"` | no | +| cluster_delete_timeout | Timeout value when deleting the EKS cluster. | string | `"15m"` | no | +| cluster_enabled_log_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | list(string) | `[]` | no | +| cluster_endpoint_private_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | bool | `"false"` | no | +| cluster_endpoint_public_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | bool | `"true"` | no | +| cluster_endpoint_public_access_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | list(string) | `[ "0.0.0.0/0" ]` | no | +| cluster_iam_role_name | IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. | string | `""` | no | +| cluster_log_kms_key_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | string | `""` | no | +| cluster_log_retention_in_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | +| cluster_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | +| cluster_security_group_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | string | `""` | no | +| cluster_version | Kubernetes version to use for the EKS cluster. | string | `"1.14"` | no | +| config_output_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | string | `"./"` | no | +| create_eks | Controls if EKS resources should be created (it affects almost all resources) | bool | `"true"` | no | +| eks_oidc_root_ca_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | string | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | +| enable_irsa | Whether to create OpenID Connect Provider for EKS to enable IRSA | bool | `"false"` | no | +| iam_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | +| kubeconfig_aws_authenticator_additional_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | +| kubeconfig_aws_authenticator_command | Command to use to fetch AWS EKS credentials. | string | `"aws-iam-authenticator"` | no | +| kubeconfig_aws_authenticator_command_args | Default arguments passed to the authenticator command. Defaults to [token -i $cluster_name]. | list(string) | `[]` | no | +| kubeconfig_aws_authenticator_env_variables | Environment variables that should be used when executing the authenticator. e.g. { AWS_PROFILE = "eks"}. | map(string) | `{}` | no | +| kubeconfig_name | Override the default name used for items kubeconfig. | string | `""` | no | +| local_exec_interpreter | Command to run for local-exec resources. Must be a shell-style interpreter. If you are on Windows Git Bash is a good choice. | list(string) | `[ "/bin/sh", "-c" ]` | no | +| manage_aws_auth | Whether to apply the aws-auth configmap file. | string | `"true"` | no | +| manage_cluster_iam_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | bool | `"true"` | no | +| manage_worker_autoscaling_policy | Whether to let the module manage the cluster autoscaling iam policy. | bool | `"true"` | no | +| manage_worker_iam_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | bool | `"true"` | no | +| map_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | +| map_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | +| map_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | +| node_groups | Map of map of node groups to create. See `node_groups` module's documentation for more details | any | `{}` | no | +| node_groups_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentaton for more details | any | `{}` | no | +| permissions_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `"null"` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | | tags | A map of tags to add to all resources. | map(string) | `{}` | no | -| vpc\_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | -| worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | -| worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | -| worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | -| worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"602401143452"` | no | -| worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"801119661308"` | no | -| worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | bool | `"false"` | no | -| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | -| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | -| worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | string | `""` | no | -| worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | number | `"1025"` | no | -| workers\_additional\_policies | Additional policies to be added to workers | list(string) | `[]` | no | -| workers\_group\_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | any | `{}` | no | -| workers\_role\_name | User defined workers role name. | string | `""` | no | -| write\_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | bool | `"true"` | no | +| vpc_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | +| worker_additional_security_group_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | +| worker_ami_name_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | +| worker_ami_name_filter_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | +| worker_ami_owner_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"602401143452"` | no | +| worker_ami_owner_id_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"801119661308"` | no | +| worker_create_initial_lifecycle_hooks | Whether to create initial lifecycle hooks provided in worker groups. | bool | `"false"` | no | +| worker_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | +| worker_groups_launch_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | +| worker_security_group_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | string | `""` | no | +| worker_sg_ingress_from_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | number | `"1025"` | no | +| workers_additional_policies | Additional policies to be added to workers | list(string) | `[]` | no | +| workers_group_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | any | `{}` | no | +| workers_role_name | User defined workers role name. | string | `""` | no | +| write_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | bool | `"true"` | no | ## Outputs | Name | Description | |------|-------------| -| cloudwatch\_log\_group\_name | Name of cloudwatch log group created | -| cluster\_arn | The Amazon Resource Name (ARN) of the cluster. | -| cluster\_certificate\_authority\_data | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | -| cluster\_endpoint | The endpoint for your EKS Kubernetes API. | -| cluster\_iam\_role\_arn | IAM role ARN of the EKS cluster. | -| cluster\_iam\_role\_name | IAM role name of the EKS cluster. | -| cluster\_id | The name/id of the EKS cluster. | -| cluster\_oidc\_issuer\_url | The URL on the EKS cluster OIDC Issuer | -| cluster\_security\_group\_id | Security group ID attached to the EKS cluster. | -| cluster\_version | The Kubernetes server version for the EKS cluster. | -| config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | +| cloudwatch_log_group_name | Name of cloudwatch log group created | +| cluster_arn | The Amazon Resource Name (ARN) of the cluster. | +| cluster_certificate_authority_data | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | +| cluster_endpoint | The endpoint for your EKS Kubernetes API. | +| cluster_iam_role_arn | IAM role ARN of the EKS cluster. | +| cluster_iam_role_name | IAM role name of the EKS cluster. | +| cluster_id | The name/id of the EKS cluster. | +| cluster_oidc_issuer_url | The URL on the EKS cluster OIDC Issuer | +| cluster_security_group_id | Security group ID attached to the EKS cluster. | +| cluster_version | The Kubernetes server version for the EKS cluster. | +| config_map_aws_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | -| kubeconfig\_filename | The filename of the generated kubectl config. | -| node\_groups | Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys | -| oidc\_provider\_arn | The ARN of the OIDC Provider if `enable_irsa = true`. | -| worker\_autoscaling\_policy\_arn | ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | -| worker\_autoscaling\_policy\_name | Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | -| worker\_iam\_instance\_profile\_arns | default IAM instance profile ARN for EKS worker groups | -| worker\_iam\_instance\_profile\_names | default IAM instance profile name for EKS worker groups | -| worker\_iam\_role\_arn | default IAM role ARN for EKS worker groups | -| worker\_iam\_role\_name | default IAM role name for EKS worker groups | -| worker\_security\_group\_id | Security group ID attached to the EKS workers. | -| workers\_asg\_arns | IDs of the autoscaling groups containing workers. | -| workers\_asg\_names | Names of the autoscaling groups containing workers. | -| workers\_default\_ami\_id | ID of the default worker group AMI | -| workers\_launch\_template\_arns | ARNs of the worker launch templates. | -| workers\_launch\_template\_ids | IDs of the worker launch templates. | -| workers\_launch\_template\_latest\_versions | Latest versions of the worker launch templates. | -| workers\_user\_data | User data of worker groups | +| kubeconfig_filename | The filename of the generated kubectl config. | +| node_groups | Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys | +| oidc_provider_arn | The ARN of the OIDC Provider if `enable_irsa = true`. | +| worker_autoscaling_policy_arn | ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | +| worker_autoscaling_policy_name | Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | +| worker_iam_instance_profile_arns | default IAM instance profile ARN for EKS worker groups | +| worker_iam_instance_profile_names | default IAM instance profile name for EKS worker groups | +| worker_iam_role_arn | default IAM role ARN for EKS worker groups | +| worker_iam_role_name | default IAM role name for EKS worker groups | +| worker_security_group_id | Security group ID attached to the EKS workers. | +| workers_asg_arns | IDs of the autoscaling groups containing workers. | +| workers_asg_names | Names of the autoscaling groups containing workers. | +| workers_default_ami_id | ID of the default worker group AMI | +| workers_launch_template_arns | ARNs of the worker launch templates. | +| workers_launch_template_ids | IDs of the worker launch templates. | +| workers_launch_template_latest_versions | Latest versions of the worker launch templates. | +| workers_user_data | User data of worker groups | diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 76278d9dc4..62ecba3e71 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -37,19 +37,19 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster\_name | Name of parent cluster | string | n/a | yes | -| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | bool | `"true"` | no | -| default\_iam\_role\_arn | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | string | n/a | yes | -| node\_groups | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | any | `{}` | no | -| node\_groups\_defaults | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | any | n/a | yes | +| cluster_name | Name of parent cluster | string | n/a | yes | +| create_eks | Controls if EKS resources should be created (it affects almost all resources) | bool | `"true"` | no | +| default_iam_role_arn | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | string | n/a | yes | +| node_groups | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | any | `{}` | no | +| node_groups_defaults | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | any | n/a | yes | | tags | A map of tags to add to all resources | map(string) | n/a | yes | -| workers\_group\_defaults | Workers group defaults from parent | any | n/a | yes | +| workers_group_defaults | Workers group defaults from parent | any | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| aws\_auth\_roles | Roles for use in aws-auth ConfigMap | -| node\_groups | Outputs from EKS node groups. Map of maps, keyed by `var.node_groups` keys. See `aws_eks_node_group` Terraform documentation for values | +| aws_auth_roles | Roles for use in aws-auth ConfigMap | +| node_groups | Outputs from EKS node groups. Map of maps, keyed by `var.node_groups` keys. See `aws_eks_node_group` Terraform documentation for values | From b70a82887717eee856b0873e0561a1328c0e7b7a Mon Sep 17 00:00:00 2001 From: Amit Thakkar Date: Wed, 15 Jan 2020 08:05:10 +0100 Subject: [PATCH 121/962] Correcting name of module (#666) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f1b2c79c1b..dd2b2902c2 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,11 @@ A full example leveraging other community modules is contained in the [examples/ ```hcl data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id + name = module.my-cluster.cluster_id } data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id + name = module.my-cluster.cluster_id } provider "kubernetes" { From 0ee36d0aee2593a33b7feb304e82e9a5b373e3ab Mon Sep 17 00:00:00 2001 From: Steffen Tautenhahn Date: Wed, 15 Jan 2020 11:46:13 +0100 Subject: [PATCH 122/962] Don't fail on destroy, when provider resource was removed (#667) * Don't fail on destroy, when provider resource was removed * Update Changelog * Node groups submodule (#650) * WIP Move node_groups to a submodule * Split the old node_groups file up * Start moving locals * Simplify IAM creation logic * depends_on from the TF docs * Wire in the variables * Call module from parent * Allow to customize the role name. As per workers * aws_auth ConfigMap for node_groups * Get the managed_node_groups example to plan * Get the basic example to plan too * create_eks = false works "The true and false result expressions must have consistent types. The given expressions are object and object, respectively." Well, that's useful. But apparently set(string) and set() are ok. So everything else is more complicated. Thanks. * Update Changelog * Update README * Wire in node_groups_defaults * Remove node_groups from workers_defaults_defaults * Synchronize random and node_group defaults * Error: "name_prefix" cannot be longer than 32 * Update READMEs again * Fix double destroy Was producing index errors when running destroy on an empty state. * Remove duplicate iam_role in node_group I think this logic works. Needs some testing with an externally created role. * Fix index fail if node group manually deleted * Keep aws_auth template in top module Downside: count causes issues as usual: can't use distinct() in the child module so there's a template render for every node_group even if only one role is really in use. Hopefully just output noise instead of technical issue * Hack to have node_groups depend on aws_auth etc The AWS Node Groups create or edit the aws-auth ConfigMap so that nodes can join the cluster. This breaks the kubernetes resource which cannot do a force create. Remove the race condition with explicit depend. Can't pull the IAM role out of the node_group any more. * Pull variables via the random_pet to cut logic No point having the same logic in two different places * Pass all ForceNew variables through the pet * Do a deep merge of NG labels and tags * Update README.. again * Additional managed node outputs #644 Add change from @TBeijin from PR #644 * Remove unused local * Use more for_each * Remove the change when create_eks = false * Make documentation less confusing * node_group version user configurable * Pass through raw output from aws_eks_node_groups * Merge workers defaults in the locals This simplifies the random_pet and aws_eks_node_group logic. Which was causing much consernation on the PR. * Fix typo Co-authored-by: Max Williams * Update Changelog * Add public access endpoint CIDRs option (terraform-aws-eks#647) (#673) * Add public access endpoint CIDRs option (terraform-aws-eks#647) * Update required provider version to 2.44.0 * Fix formatting in docs * Re-generate docs with terraform-docs 0.7.0 and bump pre-commit-terraform version (#668) * re-generate docs with terraform-docs 0.7.0 * bump pre-commit-terraform version * Release 8.0.0 (#662) * Release 8.0.0 * Update changelog * remove 'defauls' node group * Make curl silent * Update Changelog Co-authored-by: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Co-authored-by: Max Williams Co-authored-by: Siddarth Prakash <1428486+sidprak@users.noreply.github.com> Co-authored-by: Thierno IB. BARRY --- CHANGELOG.md | 1 + outputs.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63cf796f17..890eea5465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...HEAD)] - 2019-12-11] - Write your awesome change here (by @you) +- Fix index reference on destroy for output `oidc_provider_arn` (@stevie-) - Add support for restricting access to the public API endpoint (@sidprak) # History diff --git a/outputs.tf b/outputs.tf index e72b29457e..59a4077569 100644 --- a/outputs.tf +++ b/outputs.tf @@ -65,7 +65,7 @@ output "kubeconfig_filename" { output "oidc_provider_arn" { description = "The ARN of the OIDC Provider if `enable_irsa = true`." - value = var.enable_irsa ? aws_iam_openid_connect_provider.oidc_provider[0].arn : null + value = var.enable_irsa ? concat(aws_iam_openid_connect_provider.oidc_provider[*].arn, [""])[0] : null } output "workers_asg_arns" { From b3a90aac807db10561cfa39cee15d161819e7877 Mon Sep 17 00:00:00 2001 From: David Alger Date: Fri, 17 Jan 2020 07:13:07 -0600 Subject: [PATCH 123/962] Ignore changes to desired_size of node_groups (#691) * Ignore changes to desired_size of node_groups Resolves #681 * Update CHANGELOG.md --- CHANGELOG.md | 1 + modules/node_groups/node_groups.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 890eea5465..c1b8394748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Write your awesome change here (by @you) - Fix index reference on destroy for output `oidc_provider_arn` (@stevie-) - Add support for restricting access to the public API endpoint (@sidprak) +- Add an `ignore_lifecycle` rule to prevent Terraform from scaling down ASG behind AWS EKS Managed Node Group (by @davidalger) # History diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index cdbc6d00b3..e42a4ee6c4 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -45,5 +45,6 @@ resource "aws_eks_node_group" "workers" { lifecycle { create_before_destroy = true + ignore_changes = [scaling_config.0.desired_size] } } From c9b9c96cfe3cfd5827b34a4fb6a23e5cae0db860 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Fri, 17 Jan 2020 16:06:50 +0100 Subject: [PATCH 124/962] new release v8.1.0 (#693) --- CHANGELOG.md | 9 ++++++--- version | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1b8394748..bcfcd28729 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,15 +7,18 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...HEAD)] - 2019-12-11] +## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...HEAD)] - 2019-12-11] - Write your awesome change here (by @you) + +# History + +## [[v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0)] - 2019-12-11] + - Fix index reference on destroy for output `oidc_provider_arn` (@stevie-) - Add support for restricting access to the public API endpoint (@sidprak) - Add an `ignore_lifecycle` rule to prevent Terraform from scaling down ASG behind AWS EKS Managed Node Group (by @davidalger) -# History - ## [[v8.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v7.0.1)] - 2019-12-11] - **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) diff --git a/version b/version index 5f4f91fb4f..74cccb3272 100644 --- a/version +++ b/version @@ -1 +1 @@ -v8.0.0 +v8.1.0 From 317b9481ad74ce849958ab1c045f74e7ba303de9 Mon Sep 17 00:00:00 2001 From: xM8WVqaG <44505331+xM8WVqaG@users.noreply.github.com> Date: Fri, 17 Jan 2020 16:35:06 +0000 Subject: [PATCH 125/962] Update CHANGELOG.md (#695) --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcfcd28729..17c06d8e80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,19 +7,19 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...HEAD)] - 2019-12-11] +## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...HEAD)] - YYYY-MM-DD] - Write your awesome change here (by @you) # History -## [[v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0)] - 2019-12-11] +## [[v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0)] - 2020-01-17] - Fix index reference on destroy for output `oidc_provider_arn` (@stevie-) - Add support for restricting access to the public API endpoint (@sidprak) - Add an `ignore_lifecycle` rule to prevent Terraform from scaling down ASG behind AWS EKS Managed Node Group (by @davidalger) -## [[v8.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v7.0.1)] - 2019-12-11] +## [[v8.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v7.0.1)] - 2020-01-09] - **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) - **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. See Important notes below for upgrade notes (by @sdehaes) From 905d9f05a9c7253aefdcb219f14fd57c2d73035c Mon Sep 17 00:00:00 2001 From: Sanjeev Giri Date: Mon, 27 Jan 2020 10:29:24 -0500 Subject: [PATCH 126/962] Configurable local exec command for waiting until cluster is healthy (#701) * Configurable local exec command for waiting until cluster is healthy * readme * line feeds * format * fix readme * fix readme * Configurable local exec command for waiting until cluster is healthy (#1) * Configurable local exec command for waiting until cluster is healthy * readme * line feeds * format * fix readme * fix readme * change log * Configurable local exec wait 4 cluster op (#2) * Configurable local exec command for waiting until cluster is healthy * readme * line feeds * format * fix readme * fix readme * change log * changelog (#3) * Changelog (#4) * changelog * changelog * simplify wait_for_cluster command * readme * no op for manage auth false * formatting * docs? not sure * linter * specify dependency to wait for cluster more accurately --- CHANGELOG.md | 2 +- README.md | 2 +- aws_auth.tf | 2 +- cluster.tf | 16 +++++++++++++--- variables.tf | 8 ++++---- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17c06d8e80..bffbd35972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...HEAD)] - YYYY-MM-DD] -- Write your awesome change here (by @you) +- Include ability to configure custom os-specific command for waiting until kube cluster is healthy (@sanjeevgiri) # History diff --git a/README.md b/README.md index dd2b2902c2..744850a92f 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | kubeconfig_aws_authenticator_command_args | Default arguments passed to the authenticator command. Defaults to [token -i $cluster_name]. | list(string) | `[]` | no | | kubeconfig_aws_authenticator_env_variables | Environment variables that should be used when executing the authenticator. e.g. { AWS_PROFILE = "eks"}. | map(string) | `{}` | no | | kubeconfig_name | Override the default name used for items kubeconfig. | string | `""` | no | -| local_exec_interpreter | Command to run for local-exec resources. Must be a shell-style interpreter. If you are on Windows Git Bash is a good choice. | list(string) | `[ "/bin/sh", "-c" ]` | no | | manage_aws_auth | Whether to apply the aws-auth configmap file. | string | `"true"` | no | | manage_cluster_iam_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | bool | `"true"` | no | | manage_worker_autoscaling_policy | Whether to let the module manage the cluster autoscaling iam policy. | bool | `"true"` | no | @@ -188,6 +187,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | | tags | A map of tags to add to all resources. | map(string) | `{}` | no | | vpc_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | +| wait_for_cluster_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | string | `"until curl -k -s $ENDPOINT/healthz \u003e/dev/null; do sleep 4; done"` | no | | worker_additional_security_group_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | | worker_ami_name_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | | worker_ami_name_filter_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | diff --git a/aws_auth.tf b/aws_auth.tf index cce8f667c4..487763b68e 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -50,8 +50,8 @@ data "template_file" "node_group_arns" { } resource "kubernetes_config_map" "aws_auth" { - depends_on = [aws_eks_cluster.this] count = var.create_eks && var.manage_aws_auth ? 1 : 0 + depends_on = [null_resource.wait_for_cluster[0]] metadata { name = "aws-auth" diff --git a/cluster.tf b/cluster.tf index 877ddda5c7..ac43ee4330 100644 --- a/cluster.tf +++ b/cluster.tf @@ -32,10 +32,20 @@ resource "aws_eks_cluster" "this" { aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy, aws_cloudwatch_log_group.this ] +} + +resource "null_resource" "wait_for_cluster" { + count = var.manage_aws_auth ? 1 : 0 + + depends_on = [ + aws_eks_cluster.this[0] + ] + provisioner "local-exec" { - command = </dev/null; do sleep 4; done - EOT + command = var.wait_for_cluster_cmd + environment = { + ENDPOINT = aws_eks_cluster.this[0].endpoint + } } } diff --git a/variables.tf b/variables.tf index 92b906eadc..195cc58b47 100644 --- a/variables.tf +++ b/variables.tf @@ -198,10 +198,10 @@ variable "cluster_delete_timeout" { default = "15m" } -variable "local_exec_interpreter" { - description = "Command to run for local-exec resources. Must be a shell-style interpreter. If you are on Windows Git Bash is a good choice." - type = list(string) - default = ["/bin/sh", "-c"] +variable "wait_for_cluster_cmd" { + description = "Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT" + type = string + default = "until curl -k -s $ENDPOINT/healthz >/dev/null; do sleep 4; done" } variable "worker_create_initial_lifecycle_hooks" { From de90ff5d3a0397ca0c12289af1d363d39771e90d Mon Sep 17 00:00:00 2001 From: Andrey Zhelnin Date: Tue, 28 Jan 2020 16:34:55 +0100 Subject: [PATCH 127/962] Don't create ingress rule if worker security group exists (#715) --- CHANGELOG.md | 1 + cluster.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bffbd35972..075b612b72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...HEAD)] - YYYY-MM-DD] - Include ability to configure custom os-specific command for waiting until kube cluster is healthy (@sanjeevgiri) +- Disable creation of ingress rules if worker nodes security groups are exists (@andjelx) # History diff --git a/cluster.tf b/cluster.tf index ac43ee4330..0739f49439 100644 --- a/cluster.tf +++ b/cluster.tf @@ -74,7 +74,7 @@ resource "aws_security_group_rule" "cluster_egress_internet" { } resource "aws_security_group_rule" "cluster_https_worker_ingress" { - count = var.create_eks ? 1 : 0 + count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" security_group_id = local.cluster_security_group_id From 065158a970fd4fb6b6c05e09550ae44dbc521f57 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 28 Jan 2020 18:53:55 +0100 Subject: [PATCH 128/962] fix terraform pre-commit args for terraform-docs (#714) * fix terraform pre-commit args for terraform-docs * fix docs with terraform-docs 0.8.x Signed-off-by: Thierno BARRY * resolve confliting filesx * update homebrew formulas before install * update changelog --- .github/workflows/lint.yml | 2 +- .pre-commit-config.yaml | 3 +- CHANGELOG.md | 1 + README.md | 175 ++++++++++++++++++---------------- modules/node_groups/README.md | 27 ++++-- 5 files changed, 113 insertions(+), 95 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0c71b11d42..2ef163cdd0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@master - name: Install Deps - run: brew install pre-commit terraform-docs terraform + run: brew update && brew install pre-commit terraform-docs terraform - name: Check Docs run: pre-commit run --show-diff-on-failure --all-files terraform_docs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e73233f2ba..9381459513 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,8 @@ repos: - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.22.0 + rev: v1.24.0 hooks: - id: terraform_fmt - id: terraform_docs - args: [--args=--with-aggregate-type-defaults --no-escape] - id: terraform_validate - id: terraform_tflint diff --git a/CHANGELOG.md b/CHANGELOG.md index 075b612b72..5989529f34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Include ability to configure custom os-specific command for waiting until kube cluster is healthy (@sanjeevgiri) - Disable creation of ingress rules if worker nodes security groups are exists (@andjelx) +- [CI] Update pre-commit and re-generate docs to work with terraform-docs >= 0.8.1 (@barryib) # History diff --git a/README.md b/README.md index 744850a92f..95a2245e2d 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ module "my-cluster" { ``` ## Conditional creation -Sometimes you need to have a way to create EKS resources conditionally but Terraform does not allow to use `count` inside `module` block, so the solution is to specify argument `create_eks`. +Sometimes you need to have a way to create EKS resources conditionally but Terraform does not allow to use `count` inside `module` block, so the solution is to specify argument `create_eks`. Using this feature _and_ having `manage_aws_auth=true` (the default) requires to set up the kubernetes provider in a way that allows the data sources to not exist. @@ -146,95 +146,106 @@ Many thanks to [the contributors listed here](https://github.com/terraform-aws-m MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. +## Providers + +| Name | Version | +|------|---------| +| aws | >= 2.44.0 | +| kubernetes | >= 1.6.2 | +| local | >= 1.2 | +| null | >= 2.1 | +| random | >= 2.1 | +| template | >= 2.1 | + ## Inputs | Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| attach_worker_autoscaling_policy | Whether to attach the module managed cluster autoscaling iam policy to the default worker IAM role. This requires `manage_worker_autoscaling_policy = true` | bool | `"true"` | no | -| attach_worker_cni_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | bool | `"true"` | no | -| cluster_create_timeout | Timeout value when creating the EKS cluster. | string | `"15m"` | no | -| cluster_delete_timeout | Timeout value when deleting the EKS cluster. | string | `"15m"` | no | -| cluster_enabled_log_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | list(string) | `[]` | no | -| cluster_endpoint_private_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | bool | `"false"` | no | -| cluster_endpoint_public_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | bool | `"true"` | no | -| cluster_endpoint_public_access_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | list(string) | `[ "0.0.0.0/0" ]` | no | -| cluster_iam_role_name | IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. | string | `""` | no | -| cluster_log_kms_key_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | string | `""` | no | -| cluster_log_retention_in_days | Number of days to retain log events. Default retention - 90 days. | number | `"90"` | no | -| cluster_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | n/a | yes | -| cluster_security_group_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | string | `""` | no | -| cluster_version | Kubernetes version to use for the EKS cluster. | string | `"1.14"` | no | -| config_output_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | string | `"./"` | no | -| create_eks | Controls if EKS resources should be created (it affects almost all resources) | bool | `"true"` | no | -| eks_oidc_root_ca_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | string | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | -| enable_irsa | Whether to create OpenID Connect Provider for EKS to enable IRSA | bool | `"false"` | no | -| iam_path | If provided, all IAM roles will be created on this path. | string | `"/"` | no | -| kubeconfig_aws_authenticator_additional_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | list(string) | `[]` | no | -| kubeconfig_aws_authenticator_command | Command to use to fetch AWS EKS credentials. | string | `"aws-iam-authenticator"` | no | -| kubeconfig_aws_authenticator_command_args | Default arguments passed to the authenticator command. Defaults to [token -i $cluster_name]. | list(string) | `[]` | no | -| kubeconfig_aws_authenticator_env_variables | Environment variables that should be used when executing the authenticator. e.g. { AWS_PROFILE = "eks"}. | map(string) | `{}` | no | -| kubeconfig_name | Override the default name used for items kubeconfig. | string | `""` | no | -| manage_aws_auth | Whether to apply the aws-auth configmap file. | string | `"true"` | no | -| manage_cluster_iam_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified. | bool | `"true"` | no | -| manage_worker_autoscaling_policy | Whether to let the module manage the cluster autoscaling iam policy. | bool | `"true"` | no | -| manage_worker_iam_resources | Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers. | bool | `"true"` | no | -| map_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | -| map_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | -| map_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | -| node_groups | Map of map of node groups to create. See `node_groups` module's documentation for more details | any | `{}` | no | -| node_groups_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentaton for more details | any | `{}` | no | -| permissions_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `"null"` | no | -| subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | -| tags | A map of tags to add to all resources. | map(string) | `{}` | no | -| vpc_id | VPC where the cluster and workers will be deployed. | string | n/a | yes | -| wait_for_cluster_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | string | `"until curl -k -s $ENDPOINT/healthz \u003e/dev/null; do sleep 4; done"` | no | -| worker_additional_security_group_ids | A list of additional security group ids to attach to worker instances | list(string) | `[]` | no | -| worker_ami_name_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | -| worker_ami_name_filter_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used. | string | `""` | no | -| worker_ami_owner_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"602401143452"` | no | -| worker_ami_owner_id_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | string | `"801119661308"` | no | -| worker_create_initial_lifecycle_hooks | Whether to create initial lifecycle hooks provided in worker groups. | bool | `"false"` | no | -| worker_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | any | `[]` | no | -| worker_groups_launch_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | any | `[]` | no | -| worker_security_group_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | string | `""` | no | -| worker_sg_ingress_from_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | number | `"1025"` | no | -| workers_additional_policies | Additional policies to be added to workers | list(string) | `[]` | no | -| workers_group_defaults | Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys. | any | `{}` | no | -| workers_role_name | User defined workers role name. | string | `""` | no | -| write_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | bool | `"true"` | no | +|------|-------------|------|---------|:-----:| +| attach\_worker\_autoscaling\_policy | Whether to attach the module managed cluster autoscaling iam policy to the default worker IAM role. This requires `manage_worker_autoscaling_policy = true` | `bool` | `true` | no | +| attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | +| cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"15m"` | no | +| cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | +| cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | +| cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | +| cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | +| cluster\_endpoint\_public\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | +| cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. | `string` | `""` | no | +| cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `""` | no | +| cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | +| cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | +| cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | +| cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | `"1.14"` | no | +| config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | +| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | +| eks\_oidc\_root\_ca\_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | +| enable\_irsa | Whether to create OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no | +| iam\_path | If provided, all IAM roles will be created on this path. | `string` | `"/"` | no | +| kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | `list(string)` | `[]` | no | +| kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | `string` | `"aws-iam-authenticator"` | no | +| kubeconfig\_aws\_authenticator\_command\_args | Default arguments passed to the authenticator command. Defaults to [token -i $cluster\_name]. | `list(string)` | `[]` | no | +| kubeconfig\_aws\_authenticator\_env\_variables | Environment variables that should be used when executing the authenticator. e.g. { AWS\_PROFILE = "eks"}. | `map(string)` | `{}` | no | +| kubeconfig\_name | Override the default name used for items kubeconfig. | `string` | `""` | no | +| manage\_aws\_auth | Whether to apply the aws-auth configmap file. | `bool` | `true` | no | +| manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster\_iam\_role\_name must be specified. | `bool` | `true` | no | +| manage\_worker\_autoscaling\_policy | Whether to let the module manage the cluster autoscaling iam policy. | `bool` | `true` | no | +| manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam\_instance\_profile\_name must be specified for workers. | `bool` | `true` | no | +| map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | `list(string)` | `[]` | no | +| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
| `[]` | no | +| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | +| node\_groups | Map of map of node groups to create. See `node_groups` module's documentation for more details | `any` | `{}` | no | +| node\_groups\_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentaton for more details | `any` | `{}` | no | +| permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | n/a | yes | +| subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | +| tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | +| vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | +| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"until curl -k -s $ENDPOINT/healthz \u003e/dev/null; do sleep 4; done"` | no | +| worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | +| worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | +| worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | +| worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"602401143452"` | no | +| worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"801119661308"` | no | +| worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | `bool` | `false` | no | +| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | +| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | +| worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | +| worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | `number` | `1025` | no | +| workers\_additional\_policies | Additional policies to be added to workers | `list(string)` | `[]` | no | +| workers\_group\_defaults | Override default values for target groups. See workers\_group\_defaults\_defaults in local.tf for valid keys. | `any` | `{}` | no | +| workers\_role\_name | User defined workers role name. | `string` | `""` | no | +| write\_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | `bool` | `true` | no | ## Outputs | Name | Description | |------|-------------| -| cloudwatch_log_group_name | Name of cloudwatch log group created | -| cluster_arn | The Amazon Resource Name (ARN) of the cluster. | -| cluster_certificate_authority_data | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | -| cluster_endpoint | The endpoint for your EKS Kubernetes API. | -| cluster_iam_role_arn | IAM role ARN of the EKS cluster. | -| cluster_iam_role_name | IAM role name of the EKS cluster. | -| cluster_id | The name/id of the EKS cluster. | -| cluster_oidc_issuer_url | The URL on the EKS cluster OIDC Issuer | -| cluster_security_group_id | Security group ID attached to the EKS cluster. | -| cluster_version | The Kubernetes server version for the EKS cluster. | -| config_map_aws_auth | A kubernetes configuration to authenticate to this EKS cluster. | +| cloudwatch\_log\_group\_name | Name of cloudwatch log group created | +| cluster\_arn | The Amazon Resource Name (ARN) of the cluster. | +| cluster\_certificate\_authority\_data | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | +| cluster\_endpoint | The endpoint for your EKS Kubernetes API. | +| cluster\_iam\_role\_arn | IAM role ARN of the EKS cluster. | +| cluster\_iam\_role\_name | IAM role name of the EKS cluster. | +| cluster\_id | The name/id of the EKS cluster. | +| cluster\_oidc\_issuer\_url | The URL on the EKS cluster OIDC Issuer | +| cluster\_security\_group\_id | Security group ID attached to the EKS cluster. | +| cluster\_version | The Kubernetes server version for the EKS cluster. | +| config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | -| kubeconfig_filename | The filename of the generated kubectl config. | -| node_groups | Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys | -| oidc_provider_arn | The ARN of the OIDC Provider if `enable_irsa = true`. | -| worker_autoscaling_policy_arn | ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | -| worker_autoscaling_policy_name | Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | -| worker_iam_instance_profile_arns | default IAM instance profile ARN for EKS worker groups | -| worker_iam_instance_profile_names | default IAM instance profile name for EKS worker groups | -| worker_iam_role_arn | default IAM role ARN for EKS worker groups | -| worker_iam_role_name | default IAM role name for EKS worker groups | -| worker_security_group_id | Security group ID attached to the EKS workers. | -| workers_asg_arns | IDs of the autoscaling groups containing workers. | -| workers_asg_names | Names of the autoscaling groups containing workers. | -| workers_default_ami_id | ID of the default worker group AMI | -| workers_launch_template_arns | ARNs of the worker launch templates. | -| workers_launch_template_ids | IDs of the worker launch templates. | -| workers_launch_template_latest_versions | Latest versions of the worker launch templates. | -| workers_user_data | User data of worker groups | +| kubeconfig\_filename | The filename of the generated kubectl config. | +| node\_groups | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys | +| oidc\_provider\_arn | The ARN of the OIDC Provider if `enable_irsa = true`. | +| worker\_autoscaling\_policy\_arn | ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | +| worker\_autoscaling\_policy\_name | Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | +| worker\_iam\_instance\_profile\_arns | default IAM instance profile ARN for EKS worker groups | +| worker\_iam\_instance\_profile\_names | default IAM instance profile name for EKS worker groups | +| worker\_iam\_role\_arn | default IAM role ARN for EKS worker groups | +| worker\_iam\_role\_name | default IAM role name for EKS worker groups | +| worker\_security\_group\_id | Security group ID attached to the EKS workers. | +| workers\_asg\_arns | IDs of the autoscaling groups containing workers. | +| workers\_asg\_names | Names of the autoscaling groups containing workers. | +| workers\_default\_ami\_id | ID of the default worker group AMI | +| workers\_launch\_template\_arns | ARNs of the worker launch templates. | +| workers\_launch\_template\_ids | IDs of the worker launch templates. | +| workers\_launch\_template\_latest\_versions | Latest versions of the worker launch templates. | +| workers\_user\_data | User data of worker groups | diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 62ecba3e71..5681ff769e 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -33,23 +33,30 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | version | Kubernetes version | string | Provider default behavior | +## Providers + +| Name | Version | +|------|---------| +| aws | n/a | +| random | n/a | + ## Inputs | Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| cluster_name | Name of parent cluster | string | n/a | yes | -| create_eks | Controls if EKS resources should be created (it affects almost all resources) | bool | `"true"` | no | -| default_iam_role_arn | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | string | n/a | yes | -| node_groups | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | any | `{}` | no | -| node_groups_defaults | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | any | n/a | yes | -| tags | A map of tags to add to all resources | map(string) | n/a | yes | -| workers_group_defaults | Workers group defaults from parent | any | n/a | yes | +|------|-------------|------|---------|:-----:| +| cluster\_name | Name of parent cluster | `string` | n/a | yes | +| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | +| default\_iam\_role\_arn | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | n/a | yes | +| node\_groups | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | +| node\_groups\_defaults | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | n/a | yes | +| tags | A map of tags to add to all resources | `map(string)` | n/a | yes | +| workers\_group\_defaults | Workers group defaults from parent | `any` | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| aws_auth_roles | Roles for use in aws-auth ConfigMap | -| node_groups | Outputs from EKS node groups. Map of maps, keyed by `var.node_groups` keys. See `aws_eks_node_group` Terraform documentation for values | +| aws\_auth\_roles | Roles for use in aws-auth ConfigMap | +| node\_groups | Outputs from EKS node groups. Map of maps, keyed by `var.node_groups` keys. See `aws_eks_node_group` Terraform documentation for values | From dd06c591b5e8b43aecccc560447708686acacb31 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Wed, 29 Jan 2020 21:44:49 +0100 Subject: [PATCH 129/962] New release v8.2.0 (#720) --- CHANGELOG.md | 10 +++++++--- version | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5989529f34..35cc46cd12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,14 +7,18 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...HEAD)] - YYYY-MM-DD] +## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...HEAD)] - 2020-xx-xx] + +- Write your awesome change here (by @you) + +# History + +## [[v8.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...v8.2.0)] - 2020-01-29] - Include ability to configure custom os-specific command for waiting until kube cluster is healthy (@sanjeevgiri) - Disable creation of ingress rules if worker nodes security groups are exists (@andjelx) - [CI] Update pre-commit and re-generate docs to work with terraform-docs >= 0.8.1 (@barryib) -# History - ## [[v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0)] - 2020-01-17] - Fix index reference on destroy for output `oidc_provider_arn` (@stevie-) diff --git a/version b/version index 74cccb3272..7c330f2dc9 100644 --- a/version +++ b/version @@ -1 +1 @@ -v8.1.0 +v8.2.0 From cca389ababe8fd43b705b629d7972560f4efde9e Mon Sep 17 00:00:00 2001 From: Max Williams Date: Thu, 30 Jan 2020 12:27:35 +0100 Subject: [PATCH 130/962] Switch Validate github action to use env vars (#718) * Switch Validate github action to use env vars * update changelog after release * Update CHANGELOG.md Co-Authored-By: Thierno IB. BARRY Co-authored-by: Thierno IB. BARRY --- .github/workflows/lint.yml | 6 +++++- CHANGELOG.md | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2ef163cdd0..57e27e7cc4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -40,9 +40,13 @@ jobs: env: AWS_REGION: 'us-east-1' TF_WARN_OUTPUT_ERRORS: 1 + TF_VAR_vpc_id: 'vpc-123456' + TF_VAR_subnets: '["subnet-12345a"]' + TF_VAR_workers_ami_id: 'ami-123456' + TF_VAR_cluster_name: 'test_cluster' run: | terraform init - terraform validate -var "region=${AWS_REGION}" -var "vpc_id=vpc-123456" -var "subnets=[\"subnet-12345a\"]" -var "workers_ami_id=ami-123456" -var "cluster_ingress_cidrs=[]" -var "cluster_name=test_cluster" + terraform validate - name: Validate Examples run: | for example in $(find examples -maxdepth 1 -mindepth 1 -type d); do diff --git a/CHANGELOG.md b/CHANGELOG.md index 35cc46cd12..b472d5703b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...HEAD)] - 2020-xx-xx] - Write your awesome change here (by @you) +- [CI] Switch `Validate` github action to use env vars (by @max-rocket-internet) # History From 9032dce006bc381d0884cb6b5e48803126954a04 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Thu, 30 Jan 2020 15:21:20 +0100 Subject: [PATCH 131/962] Add example for IRSA and cluster-autoscaler (#710) * Add example for IRSA * remove unused SGs * remove helm setup part * simplification and use module * fix fmt error * remove unused resource --- CHANGELOG.md | 1 + examples/irsa/README.md | 63 ++++++++++++++ .../irsa/cluster-autoscaler-chart-values.yaml | 10 +++ examples/irsa/irsa.tf | 57 +++++++++++++ examples/irsa/locals.tf | 5 ++ examples/irsa/main.tf | 82 +++++++++++++++++++ examples/irsa/outputs.tf | 3 + examples/irsa/variables.tf | 3 + 8 files changed, 224 insertions(+) create mode 100644 examples/irsa/README.md create mode 100644 examples/irsa/cluster-autoscaler-chart-values.yaml create mode 100644 examples/irsa/irsa.tf create mode 100644 examples/irsa/locals.tf create mode 100644 examples/irsa/main.tf create mode 100644 examples/irsa/outputs.tf create mode 100644 examples/irsa/variables.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index b472d5703b..cf088d2fd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Include ability to configure custom os-specific command for waiting until kube cluster is healthy (@sanjeevgiri) - Disable creation of ingress rules if worker nodes security groups are exists (@andjelx) - [CI] Update pre-commit and re-generate docs to work with terraform-docs >= 0.8.1 (@barryib) +- Added example `examples/irsa` for IAM Roles for Service Accounts (by @max-rocket-internet) ## [[v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0)] - 2020-01-17] diff --git a/examples/irsa/README.md b/examples/irsa/README.md new file mode 100644 index 0000000000..0af4d7b440 --- /dev/null +++ b/examples/irsa/README.md @@ -0,0 +1,63 @@ +# IAM Roles for Service Accounts + +This example shows how to create an IAM role to be used for a Kubernetes `ServiceAccount`. It will create a policy and role to be used by the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) using the [public Helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler). + +The AWS documentation for IRSA is here: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html + +## Setup + +Run Terraform: + +``` +terraform init +terraform apply +``` + +Set kubectl context to the new cluster: `export KUBECONFIG=kubeconfig_test-eks-irsa` + +Check that there is a node that is `Ready`: + +``` +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +ip-10-0-2-190.us-west-2.compute.internal Ready 6m39s v1.14.8-eks-b8860f +``` + +Replace `` with your AWS account ID in `cluster-autoscaler-chart-values.yaml`. There is output from terraform for this. + +Install the chart using the provided values file: + +``` +helm install --name cluster-autoscaler --namespace kube-system stable/cluster-autoscaler --values=cluster-autoscaler-chart-values.yaml +``` + +## Verify + +Ensure the cluster-autoscaler pod is running: + +``` +$ kubectl --namespace=kube-system get pods -l "app.kubernetes.io/name=aws-cluster-autoscaler" +NAME READY STATUS RESTARTS AGE +cluster-autoscaler-aws-cluster-autoscaler-5545d4b97-9ztpm 1/1 Running 0 3m +``` + +Observe the `AWS_*` environment variables that were added to the pod automatically by EKS: + +``` +kubectl --namespace=kube-system get pods -l "app.kubernetes.io/name=aws-cluster-autoscaler" -o yaml | grep -A3 AWS_ROLE_ARN + +- name: AWS_ROLE_ARN + value: arn:aws:iam::xxxxxxxxx:role/cluster-autoscaler +- name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +``` + +Verify it is working by checking the logs, you should see that it has discovered the autoscaling group successfully: + +``` +kubectl --namespace=kube-system logs -l "app.kubernetes.io/name=aws-cluster-autoscaler" + +I0128 14:59:00.901513 1 auto_scaling_groups.go:354] Regenerating instance to ASG map for ASGs: [test-eks-irsa-worker-group-12020012814125354700000000e] +I0128 14:59:00.969875 1 auto_scaling_groups.go:138] Registering ASG test-eks-irsa-worker-group-12020012814125354700000000e +I0128 14:59:00.969906 1 aws_manager.go:263] Refreshed ASG list, next refresh after 2020-01-28 15:00:00.969901767 +0000 UTC m=+61.310501783 +``` diff --git a/examples/irsa/cluster-autoscaler-chart-values.yaml b/examples/irsa/cluster-autoscaler-chart-values.yaml new file mode 100644 index 0000000000..71b18c43a0 --- /dev/null +++ b/examples/irsa/cluster-autoscaler-chart-values.yaml @@ -0,0 +1,10 @@ +awsRegion: us-west-2 + +rbac: + create: true + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: "arn:aws:iam:::role/cluster-autoscaler" + +autoDiscovery: + clusterName: test-eks-irsa + enabled: true diff --git a/examples/irsa/irsa.tf b/examples/irsa/irsa.tf new file mode 100644 index 0000000000..8ba8f06823 --- /dev/null +++ b/examples/irsa/irsa.tf @@ -0,0 +1,57 @@ +module "iam_assumable_role_admin" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "~> v2.6.0" + create_role = true + role_name = "cluster-autoscaler" + provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") + role_policy_arns = [aws_iam_policy.cluster_autoscaler.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:${local.k8s_service_account_namespace}:${local.k8s_service_account_name}"] +} + +resource "aws_iam_policy" "cluster_autoscaler" { + name_prefix = "cluster-autoscaler" + description = "EKS cluster-autoscaler policy for cluster ${module.eks.cluster_id}" + policy = data.aws_iam_policy_document.cluster_autoscaler.json +} + +data "aws_iam_policy_document" "cluster_autoscaler" { + statement { + sid = "clusterAutoscalerAll" + effect = "Allow" + + actions = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions", + ] + + resources = ["*"] + } + + statement { + sid = "clusterAutoscalerOwn" + effect = "Allow" + + actions = [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup", + ] + + resources = ["*"] + + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${module.eks.cluster_id}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled" + values = ["true"] + } + } +} diff --git a/examples/irsa/locals.tf b/examples/irsa/locals.tf new file mode 100644 index 0000000000..9cdc8af713 --- /dev/null +++ b/examples/irsa/locals.tf @@ -0,0 +1,5 @@ +locals { + cluster_name = "test-eks-irsa" + k8s_service_account_namespace = "kube-system" + k8s_service_account_name = "cluster-autoscaler-aws-cluster-autoscaler" +} diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf new file mode 100644 index 0000000000..0938a0596e --- /dev/null +++ b/examples/irsa/main.tf @@ -0,0 +1,82 @@ +terraform { + required_version = ">= 0.12.0" +} + +provider "aws" { + version = ">= 2.28.1" + region = var.region +} + +provider "local" { + version = "~> 1.2" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false + version = "~> 1.10" +} + +data "aws_availability_zones" "available" {} + +data "aws_caller_identity" "current" {} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.6.0" + name = "test-vpc" + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + public_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + enable_dns_hostnames = true + + tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/elb" = "1" + } +} + +module "eks" { + source = "../.." + cluster_name = local.cluster_name + subnets = module.vpc.public_subnets + vpc_id = module.vpc.vpc_id + enable_irsa = true + + worker_groups = [ + { + name = "worker-group-1" + instance_type = "t2.medium" + asg_desired_capacity = 1 + tags = [ + { + "key" = "k8s.io/cluster-autoscaler/enabled" + "propagate_at_launch" = "false" + "value" = "true" + } + ] + } + ] +} diff --git a/examples/irsa/outputs.tf b/examples/irsa/outputs.tf new file mode 100644 index 0000000000..ef2ab9577a --- /dev/null +++ b/examples/irsa/outputs.tf @@ -0,0 +1,3 @@ +output "aws_account_id" { + value = data.aws_caller_identity.current.account_id +} diff --git a/examples/irsa/variables.tf b/examples/irsa/variables.tf new file mode 100644 index 0000000000..81b8dbe73e --- /dev/null +++ b/examples/irsa/variables.tf @@ -0,0 +1,3 @@ +variable "region" { + default = "us-west-2" +} From 92b5c2ad3fe7906b9a71f23376cde7a13348a57a Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 30 Jan 2020 15:50:56 +0100 Subject: [PATCH 132/962] bump pre-commit-terraform version (#724) --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9381459513..1b77bc2d98 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.24.0 + rev: v1.25.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/CHANGELOG.md b/CHANGELOG.md index cf088d2fd4..47eb0dcbd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Write your awesome change here (by @you) - [CI] Switch `Validate` github action to use env vars (by @max-rocket-internet) +- [CI] Bump pre-commit-terraform version (by @barryib) # History From 626a393ab964677e3b4fb1336ad9d8cef4c26406 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 4 Feb 2020 19:34:17 +0100 Subject: [PATCH 133/962] Remove of autoscaling IAM policy related stuff (#716) * Disable management of autoscaling IAM policy by default * remove completely * update changelog notes * fix pre-commit stuff * misc updates * fmt * fix changelog * Removal of tags and update docs * Change after updating terraform-docs * Add second tag --- CHANGELOG.md | 10 ++++- README.md | 10 ++--- docs/autoscaling.md | 76 +++++++++++++++++++++++++++++++----- docs/faq.md | 2 +- docs/spot-instances.md | 4 -- examples/irsa/main.tf | 5 +++ local.tf | 1 - node_groups.tf | 9 ++--- outputs.tf | 10 ----- variables.tf | 12 ------ workers.tf | 80 -------------------------------------- workers_launch_template.tf | 23 ----------- 12 files changed, 87 insertions(+), 155 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47eb0dcbd3..3a9985361b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,14 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...HEAD)] - 2020-xx-xx] -- Write your awesome change here (by @you) - [CI] Switch `Validate` github action to use env vars (by @max-rocket-internet) - [CI] Bump pre-commit-terraform version (by @barryib) +- Added example `examples/irsa` for IAM Roles for Service Accounts (by @max-rocket-internet) +- **Breaking:** Removal of autoscaling IAM policy and tags (by @max-rocket-internet) + +#### Important notes + +Autoscaling policy and tags have been removed from this module. This reduces complexity and increases security as the policy was attached to the node group IAM role. To manage it outside of this module either follow the example in `examples/irsa` to attach an IAM role to the cluster-autoscaler `serviceAccount` or create the policy outside this module and pass it in using the `workers_additional_policies` variable. # History @@ -20,7 +25,8 @@ project adheres to [Semantic Versioning](http://semver.org/). - Include ability to configure custom os-specific command for waiting until kube cluster is healthy (@sanjeevgiri) - Disable creation of ingress rules if worker nodes security groups are exists (@andjelx) - [CI] Update pre-commit and re-generate docs to work with terraform-docs >= 0.8.1 (@barryib) -- Added example `examples/irsa` for IAM Roles for Service Accounts (by @max-rocket-internet) + +# History ## [[v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0)] - 2020-01-17] diff --git a/README.md b/README.md index 95a2245e2d..8c4a916905 100644 --- a/README.md +++ b/README.md @@ -161,14 +161,13 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | Type | Default | Required | |------|-------------|------|---------|:-----:| -| attach\_worker\_autoscaling\_policy | Whether to attach the module managed cluster autoscaling iam policy to the default worker IAM role. This requires `manage_worker_autoscaling_policy = true` | `bool` | `true` | no | | attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | | cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"15m"` | no | | cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | -| cluster\_endpoint\_public\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | +| cluster\_endpoint\_public\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | | cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. | `string` | `""` | no | | cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `""` | no | | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | @@ -187,11 +186,10 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | kubeconfig\_name | Override the default name used for items kubeconfig. | `string` | `""` | no | | manage\_aws\_auth | Whether to apply the aws-auth configmap file. | `bool` | `true` | no | | manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster\_iam\_role\_name must be specified. | `bool` | `true` | no | -| manage\_worker\_autoscaling\_policy | Whether to let the module manage the cluster autoscaling iam policy. | `bool` | `true` | no | | manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam\_instance\_profile\_name must be specified for workers. | `bool` | `true` | no | | map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | `list(string)` | `[]` | no | -| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
| `[]` | no | -| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | +| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
| `[]` | no | +| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | | node\_groups | Map of map of node groups to create. See `node_groups` module's documentation for more details | `any` | `{}` | no | | node\_groups\_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentaton for more details | `any` | `{}` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | n/a | yes | @@ -233,8 +231,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | kubeconfig\_filename | The filename of the generated kubectl config. | | node\_groups | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys | | oidc\_provider\_arn | The ARN of the OIDC Provider if `enable_irsa = true`. | -| worker\_autoscaling\_policy\_arn | ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | -| worker\_autoscaling\_policy\_name | Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true` | | worker\_iam\_instance\_profile\_arns | default IAM instance profile ARN for EKS worker groups | | worker\_iam\_instance\_profile\_names | default IAM instance profile name for EKS worker groups | | worker\_iam\_role\_arn | default IAM role ARN for EKS worker groups | diff --git a/docs/autoscaling.md b/docs/autoscaling.md index 2aabab83d2..d13d40941c 100644 --- a/docs/autoscaling.md +++ b/docs/autoscaling.md @@ -1,12 +1,72 @@ # Autoscaling -Autoscaling of worker nodes can be easily enabled by setting the `autoscaling_enabled` variable to `true` for a worker group in the `worker_groups` map. -This will add the required tags to the autoscaling group for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler). -One should also set `protect_from_scale_in` to `true` for such worker groups, to ensure that cluster-autoscaler is solely responsible for scaling events. +To enable worker node autoscaling you will need to do a few things: -You will also need to install the cluster-autoscaler into your cluster. The easiest way to do this is with [helm](https://helm.sh/). +- Add the [required tags](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#auto-discovery-setup) to the worker group +- Install the cluster-autoscaler +- Give the cluster-autoscaler access via an IAM policy -The [helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler) for the cluster-autoscaler requires some specific settings to work in an EKS cluster. These settings are supplied via YAML values file when installing the helm chart. Here is an example values file: +It's probably easiest to follow the example in [examples/irsa](../examples/irsa), this will install the cluster-autoscaler using [Helm](https://helm.sh/) and use IRSA to attach a policy. + +If you don't want to use IRSA then you will need to attach the IAM policy to the worker node IAM role or add AWS credentials to the cluster-autoscaler environment variables. Here is some example terraform code for the policy: + +```hcl +resource "aws_iam_role_policy_attachment" "workers_autoscaling" { + policy_arn = aws_iam_policy.worker_autoscaling.arn + role = module.my_cluster.worker_iam_role_name[0] +} + +resource "aws_iam_policy" "worker_autoscaling" { + name_prefix = "eks-worker-autoscaling-${module.my_cluster.cluster_id}" + description = "EKS worker node autoscaling policy for cluster ${module.my_cluster.cluster_id}" + policy = data.aws_iam_policy_document.worker_autoscaling.json + path = var.iam_path +} + +data "aws_iam_policy_document" "worker_autoscaling" { + statement { + sid = "eksWorkerAutoscalingAll" + effect = "Allow" + + actions = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions", + ] + + resources = ["*"] + } + + statement { + sid = "eksWorkerAutoscalingOwn" + effect = "Allow" + + actions = [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup", + ] + + resources = ["*"] + + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${module.my_cluster.cluster_id}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled" + values = ["true"] + } + } +} +``` + +And example values for the [helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler): ```yaml rbac: @@ -26,10 +86,6 @@ To install the chart, simply run helm with the `--values` option: helm install stable/cluster-autoscaler --values=path/to/your/values-file.yaml ``` -`NOTE` +## Notes There is a variable `asg_desired_capacity` given in the `local.tf` file, currently it can be used to change the desired worker(s) capacity in the autoscaling group but currently it is being ignored in terraform to reduce the [complexities](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/510#issuecomment-531700442) and the feature of scaling up and down the cluster nodes is being handled by the cluster autoscaler. - -## See More - -[Using AutoScalingGroup MixedInstancesPolicy](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-autoscalinggroup-mixedinstancespolicy) diff --git a/docs/faq.md b/docs/faq.md index d793f28fea..34d0aa6633 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -56,7 +56,7 @@ The safest and easiest option is to set `asg_min_size` and `asg_max_size` to 0 o The module is configured to ignore this value. Unfortunately Terraform does not support variables within the `lifecycle` block. -The setting is ignored to allow the cluster autoscaler to work correctly and so that terraform applys do not accidentally remove running workers. +The setting is ignored to allow the cluster autoscaler to work correctly and so that terraform apply does not accidentally remove running workers. You can change the desired count via the CLI or console if you're not using the cluster autoscaler. diff --git a/docs/spot-instances.md b/docs/spot-instances.md index 06ca2e8068..1a5da6e79e 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -32,7 +32,6 @@ Example worker group configuration that uses an ASG with launch configuration fo name = "on-demand-1" instance_type = "m4.xlarge" asg_max_size = 1 - autoscaling_enabled = true kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=normal" suspended_processes = ["AZRebalance"] }, @@ -41,7 +40,6 @@ Example worker group configuration that uses an ASG with launch configuration fo spot_price = "0.199" instance_type = "c4.xlarge" asg_max_size = 20 - autoscaling_enabled = true kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" suspended_processes = ["AZRebalance"] }, @@ -50,7 +48,6 @@ Example worker group configuration that uses an ASG with launch configuration fo spot_price = "0.20" instance_type = "m4.xlarge" asg_max_size = 20 - autoscaling_enabled = true kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" suspended_processes = ["AZRebalance"] } @@ -67,7 +64,6 @@ Launch Template support is a recent addition to both AWS and this module. It mig name = "on-demand-1" instance_type = "m4.xlarge" asg_max_size = 10 - autoscaling_enabled = true kubelet_extra_args = "--node-labels=spot=false" suspended_processes = ["AZRebalance"] } diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index 0938a0596e..2121a589ec 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -75,6 +75,11 @@ module "eks" { "key" = "k8s.io/cluster-autoscaler/enabled" "propagate_at_launch" = "false" "value" = "true" + }, + { + "key" = "k8s.io/cluster-autoscaler/${local.cluster_name}" + "propagate_at_launch" = "false" + "value" = "true" } ] } diff --git a/local.tf b/local.tf index 609185816f..2487de3009 100644 --- a/local.tf +++ b/local.tf @@ -49,7 +49,6 @@ locals { public_ip = false # Associate a public ip address with a worker kubelet_extra_args = "" # This string is passed directly to kubelet if set. Useful for adding labels or taints. subnets = var.subnets # A list of subnets to place the worker nodes in. i.e. ["subnet-123", "subnet-456", "subnet-789"] - autoscaling_enabled = false # Sets whether policy and matching tags will be added to allow autoscaling. additional_security_group_ids = [] # A list of additional security group ids to include in worker launch config protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. diff --git a/node_groups.tf b/node_groups.tf index 6c7b438cfb..2c689c3818 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -9,11 +9,10 @@ data "null_data_source" "node_groups" { # Ensure these resources are created before "unlocking" the data source. # `depends_on` causes a refresh on every run so is useless here. # [Re]creating or removing these resources will trigger recreation of Node Group resources - aws_auth = coalescelist(kubernetes_config_map.aws_auth[*].id, [""])[0] - role_NodePolicy = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy[*].id, [""])[0] - role_CNI_Policy = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy[*].id, [""])[0] - role_Container = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly[*].id, [""])[0] - role_autoscaling = coalescelist(aws_iam_role_policy_attachment.workers_autoscaling[*].id, [""])[0] + aws_auth = coalescelist(kubernetes_config_map.aws_auth[*].id, [""])[0] + role_NodePolicy = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy[*].id, [""])[0] + role_CNI_Policy = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy[*].id, [""])[0] + role_Container = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly[*].id, [""])[0] } } diff --git a/outputs.tf b/outputs.tf index 59a4077569..b1f8c3c049 100644 --- a/outputs.tf +++ b/outputs.tf @@ -153,16 +153,6 @@ output "worker_iam_role_arn" { )[0] } -output "worker_autoscaling_policy_name" { - description = "Name of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true`" - value = concat(aws_iam_policy.worker_autoscaling[*].name, [""])[0] -} - -output "worker_autoscaling_policy_arn" { - description = "ARN of the worker autoscaling IAM policy if `manage_worker_autoscaling_policy = true`" - value = concat(aws_iam_policy.worker_autoscaling[*].arn, [""])[0] -} - output "node_groups" { description = "Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys" value = module.node_groups.node_groups diff --git a/variables.tf b/variables.tf index 195cc58b47..b9fa6c4c0a 100644 --- a/variables.tf +++ b/variables.tf @@ -264,18 +264,6 @@ variable "workers_role_name" { default = "" } -variable "manage_worker_autoscaling_policy" { - description = "Whether to let the module manage the cluster autoscaling iam policy." - type = bool - default = true -} - -variable "attach_worker_autoscaling_policy" { - description = "Whether to attach the module managed cluster autoscaling iam policy to the default worker IAM role. This requires `manage_worker_autoscaling_policy = true`" - type = bool - default = true -} - variable "attach_worker_cni_policy" { description = "Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster." type = bool diff --git a/workers.tf b/workers.tf index b6e6f69f9e..839452ed43 100644 --- a/workers.tf +++ b/workers.tf @@ -104,29 +104,6 @@ resource "aws_autoscaling_group" "workers" { "value" = "owned" "propagate_at_launch" = true }, - { - "key" = "k8s.io/cluster-autoscaler/${lookup( - var.worker_groups[count.index], - "autoscaling_enabled", - local.workers_group_defaults["autoscaling_enabled"], - ) ? "enabled" : "disabled"}" - "value" = "true" - "propagate_at_launch" = false - }, - { - "key" = "k8s.io/cluster-autoscaler/${aws_eks_cluster.this[0].name}" - "value" = aws_eks_cluster.this[0].name - "propagate_at_launch" = false - }, - { - "key" = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" - "value" = "${lookup( - var.worker_groups[count.index], - "root_volume_size", - local.workers_group_defaults["root_volume_size"], - )}Gi" - "propagate_at_launch" = false - }, ], local.asg_tags, lookup( @@ -357,60 +334,3 @@ resource "aws_iam_role_policy_attachment" "workers_additional_policies" { role = aws_iam_role.workers[0].name policy_arn = var.workers_additional_policies[count.index] } - -resource "aws_iam_role_policy_attachment" "workers_autoscaling" { - count = var.manage_worker_iam_resources && var.manage_worker_autoscaling_policy && var.attach_worker_autoscaling_policy && var.create_eks ? 1 : 0 - policy_arn = aws_iam_policy.worker_autoscaling[0].arn - role = aws_iam_role.workers[0].name -} - -resource "aws_iam_policy" "worker_autoscaling" { - count = var.manage_worker_iam_resources && var.manage_worker_autoscaling_policy && var.create_eks ? 1 : 0 - name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this[0].name}" - description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this[0].name}" - policy = data.aws_iam_policy_document.worker_autoscaling[0].json - path = var.iam_path -} - -data "aws_iam_policy_document" "worker_autoscaling" { - count = var.manage_worker_iam_resources && var.manage_worker_autoscaling_policy && var.create_eks ? 1 : 0 - statement { - sid = "eksWorkerAutoscalingAll" - effect = "Allow" - - actions = [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "ec2:DescribeLaunchTemplateVersions", - ] - - resources = ["*"] - } - - statement { - sid = "eksWorkerAutoscalingOwn" - effect = "Allow" - - actions = [ - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - ] - - resources = ["*"] - - condition { - test = "StringEquals" - variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled" - values = ["true"] - } - } -} diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 519a289942..ba51baaf9d 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -179,29 +179,6 @@ resource "aws_autoscaling_group" "workers_launch_template" { "value" = "owned" "propagate_at_launch" = true }, - { - "key" = "k8s.io/cluster-autoscaler/${lookup( - var.worker_groups_launch_template[count.index], - "autoscaling_enabled", - local.workers_group_defaults["autoscaling_enabled"], - ) ? "enabled" : "disabled"}" - "value" = "true" - "propagate_at_launch" = false - }, - { - "key" = "k8s.io/cluster-autoscaler/${aws_eks_cluster.this[0].name}" - "value" = aws_eks_cluster.this[0].name - "propagate_at_launch" = false - }, - { - "key" = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" - "value" = "${lookup( - var.worker_groups_launch_template[count.index], - "root_volume_size", - local.workers_group_defaults["root_volume_size"], - )}Gi" - "propagate_at_launch" = false - }, ], local.asg_tags, lookup( From 415e123d93b1ec55df1e6125881996a43ac546a6 Mon Sep 17 00:00:00 2001 From: Daniele Lisi Date: Thu, 6 Feb 2020 03:45:45 -0800 Subject: [PATCH 134/962] Add iam:GetOpenIDConnectProvider grant to docs/iam-permissions.md (#728) * Add iam:GetOpenIDConnectProvider grant to docs/iam-permissions.md * Update CHANGELOG.md --- CHANGELOG.md | 1 + docs/iam-permissions.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a9985361b..83bc914b86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - [CI] Bump pre-commit-terraform version (by @barryib) - Added example `examples/irsa` for IAM Roles for Service Accounts (by @max-rocket-internet) - **Breaking:** Removal of autoscaling IAM policy and tags (by @max-rocket-internet) +- Add `iam:GetOpenIDConnectProvider` grant to the required IAM permissions in `docs/iam-permissions.md` (by @danielelisi) #### Important notes diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index 4fbc828447..9c15083409 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -97,6 +97,7 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "iam:DeleteServiceLinkedRole", "iam:DetachRolePolicy", "iam:GetInstanceProfile", + "iam:GetOpenIDConnectProvider", "iam:GetPolicy", "iam:GetPolicyVersion", "iam:GetRole", @@ -118,4 +119,4 @@ Following IAM permissions are the minimum permissions needed for your IAM user o } ] } -``` \ No newline at end of file +``` From 6c42d23c23d501c62bac6d8f0d9039f7c46a8307 Mon Sep 17 00:00:00 2001 From: ivanguravel Date: Thu, 6 Feb 2020 18:46:38 +0200 Subject: [PATCH 135/962] Detailed description for enabling Windows nodes for AWS EKS (#727) * Update faq.md detailed algorithm for building EKS cluster with Windows nodes * small description fix * Update docs/faq.md Co-Authored-By: Max Williams * Update faq.md * Update faq.md * Update CHANGELOG.md * Update docs/faq.md Co-Authored-By: Max Williams * Update docs/faq.md Co-Authored-By: Max Williams * Update docs/faq.md Co-Authored-By: Max Williams * Update docs/faq.md Co-Authored-By: Max Williams * Update CHANGELOG.md Co-Authored-By: Max Williams Co-authored-by: Max Williams --- CHANGELOG.md | 1 + docs/faq.md | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83bc914b86..fb62a4d343 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...HEAD)] - 2020-xx-xx] +- Added instructions for how to add Windows nodes (by @ivanguravel) - [CI] Switch `Validate` github action to use env vars (by @max-rocket-internet) - [CI] Bump pre-commit-terraform version (by @barryib) - Added example `examples/irsa` for IAM Roles for Service Accounts (by @max-rocket-internet) diff --git a/docs/faq.md b/docs/faq.md index 34d0aa6633..b57af69791 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -104,3 +104,43 @@ module "eks" { To enable Windows support for your EKS cluster, you should apply some configs manually. See the [Enabling Windows Support (Windows/MacOS/Linux)](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support). Windows worker nodes requires additional cluster role (eks:kube-proxy-windows). If you are adding windows workers to existing cluster, you should apply config-map-aws-auth again. + +#### Example configuration + +Amazon EKS clusters must contain one or more Linux worker nodes to run core system pods that only run on Linux, such as coredns and the VPC resource controller. + +1. Build AWS EKS cluster with the next workers configuration (default Linux): + +``` +worker_groups = [ + { + name = "worker-group-linux" + instance_type = "m5.large" + platform = "linux" + asg_desired_capacity = 2 + }, + ] +``` + +2. Apply commands from https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support (use tab with name `Windows`) + +3. Add one more worker group for Windows with required field `platform = "windows"` and update your cluster. Worker group example: + +``` +worker_groups = [ + { + name = "worker-group-linux" + instance_type = "m5.large" + platform = "linux" + asg_desired_capacity = 2 + }, + { + name = "worker-group-windows" + instance_type = "m5.large" + platform = "windows" + asg_desired_capacity = 1 + }, + ] +``` + +4. Wtih `kubectl get nodes` you can see cluster with mixed (Linux/Windows) nodes support. From a9c73b277e8edf0681e5100ed7f42a240686ff2a Mon Sep 17 00:00:00 2001 From: Daniele Lisi Date: Fri, 7 Feb 2020 06:33:28 -0800 Subject: [PATCH 136/962] Add `iam:{Create,Delete}OpenIDProviderConnect` to required IAM policies (#729) --- CHANGELOG.md | 2 +- docs/iam-permissions.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb62a4d343..0213bb492d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - [CI] Bump pre-commit-terraform version (by @barryib) - Added example `examples/irsa` for IAM Roles for Service Accounts (by @max-rocket-internet) - **Breaking:** Removal of autoscaling IAM policy and tags (by @max-rocket-internet) -- Add `iam:GetOpenIDConnectProvider` grant to the required IAM permissions in `docs/iam-permissions.md` (by @danielelisi) +- Add `iam:{Create,Delete,Get}OpenIDConnectProvider` grants to the list of required IAM permissions in `docs/iam-permissions.md` (by @danielelisi) #### Important notes diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index 9c15083409..f0e161140c 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -86,11 +86,13 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy", "iam:CreateInstanceProfile", + "iam:CreateOpenIDConnectProvider", "iam:CreateServiceLinkedRole", "iam:CreatePolicy", "iam:CreatePolicyVersion", "iam:CreateRole", "iam:DeleteInstanceProfile", + "iam:DeleteOpenIDConnectProvider", "iam:DeletePolicy", "iam:DeleteRole", "iam:DeleteRolePolicy", From 10ca272e5b7ce006a721c99ef8906720ff5cd6ee Mon Sep 17 00:00:00 2001 From: Matthias Splieth Date: Fri, 14 Feb 2020 10:10:29 +0100 Subject: [PATCH 137/962] Add option to set the name of a node group manually (#739) --- CHANGELOG.md | 1 + modules/node_groups/README.md | 1 + modules/node_groups/node_groups.tf | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0213bb492d..3fd48e438d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Added example `examples/irsa` for IAM Roles for Service Accounts (by @max-rocket-internet) - **Breaking:** Removal of autoscaling IAM policy and tags (by @max-rocket-internet) - Add `iam:{Create,Delete,Get}OpenIDConnectProvider` grants to the list of required IAM permissions in `docs/iam-permissions.md` (by @danielelisi) +- Add an `name` parameter to be able to manually name EKS Managed Node Groups (by @splieth) #### Important notes diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 5681ff769e..6ac8842f35 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -28,6 +28,7 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` | | max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` | | min\_capacity | Min number of workers | number | `var.workers_group_defaults[asg_min_size]` | +| name | Name of the node group | string | Auto generated | | source\_security\_group\_ids | Source security groups for remote access to workers | list(string) | If key\_name is specified: THE REMOTE ACCESS WILL BE OPENED TO THE WORLD | | subnets | Subnets to contain workers | list(string) | `var.workers_group_defaults[subnets]` | | version | Kubernetes version | string | Provider default behavior | diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index e42a4ee6c4..62dc6bff98 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -1,7 +1,7 @@ resource "aws_eks_node_group" "workers" { for_each = local.node_groups_expanded - node_group_name = join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id]) + node_group_name = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) cluster_name = var.cluster_name node_role_arn = each.value["iam_role_arn"] From f2cc9f503919d5a0a4a16fe3c8336d6d155fe48f Mon Sep 17 00:00:00 2001 From: Ryan Ooi Date: Thu, 27 Feb 2020 18:23:41 +0800 Subject: [PATCH 138/962] Revert #631. Add back manage security group flag (#722) * Revert #631 * fix README lint * fix README lint for bool --- CHANGELOG.md | 1 + README.md | 2 ++ cluster.tf | 6 +++--- local.tf | 4 ++-- variables.tf | 12 ++++++++++++ workers.tf | 12 ++++++------ 6 files changed, 26 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fd48e438d..26ff3f3a1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...HEAD)] - 2020-xx-xx] +- Revert #631. Add back manage security group flags. (by @ryanooi) - Added instructions for how to add Windows nodes (by @ivanguravel) - [CI] Switch `Validate` github action to use env vars (by @max-rocket-internet) diff --git a/README.md b/README.md index 8c4a916905..97ac363496 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | Type | Default | Required | |------|-------------|------|---------|:-----:| | attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | +| cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | | cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"15m"` | no | | cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | @@ -203,6 +204,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"602401143452"` | no | | worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"801119661308"` | no | | worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | `bool` | `false` | no | +| worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | `bool` | `true` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | | worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | diff --git a/cluster.tf b/cluster.tf index 0739f49439..c5439adea7 100644 --- a/cluster.tf +++ b/cluster.tf @@ -50,7 +50,7 @@ resource "null_resource" "wait_for_cluster" { } resource "aws_security_group" "cluster" { - count = var.cluster_security_group_id == "" && var.create_eks ? 1 : 0 + count = var.cluster_create_security_group && var.create_eks ? 1 : 0 name_prefix = var.cluster_name description = "EKS cluster security group." vpc_id = var.vpc_id @@ -63,7 +63,7 @@ resource "aws_security_group" "cluster" { } resource "aws_security_group_rule" "cluster_egress_internet" { - count = var.cluster_security_group_id == "" && var.create_eks ? 1 : 0 + count = var.cluster_create_security_group && var.create_eks ? 1 : 0 description = "Allow cluster egress access to the Internet." protocol = "-1" security_group_id = local.cluster_security_group_id @@ -74,7 +74,7 @@ resource "aws_security_group_rule" "cluster_egress_internet" { } resource "aws_security_group_rule" "cluster_https_worker_ingress" { - count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 + count = var.cluster_create_security_group && var.create_eks ? 1 : 0 description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" security_group_id = local.cluster_security_group_id diff --git a/local.tf b/local.tf index 2487de3009..8ed32f8541 100644 --- a/local.tf +++ b/local.tf @@ -8,10 +8,10 @@ locals { ) ] - cluster_security_group_id = var.cluster_security_group_id == "" ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id + cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) - worker_security_group_id = var.worker_security_group_id == "" ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id + worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name diff --git a/variables.tf b/variables.tf index b9fa6c4c0a..9ba78b441a 100644 --- a/variables.tf +++ b/variables.tf @@ -204,6 +204,18 @@ variable "wait_for_cluster_cmd" { default = "until curl -k -s $ENDPOINT/healthz >/dev/null; do sleep 4; done" } +variable "cluster_create_security_group" { + description = "Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`." + type = bool + default = true +} + +variable "worker_create_security_group" { + description = "Whether to create a security group for the workers or attach the workers to `worker_security_group_id`." + type = bool + default = true +} + variable "worker_create_initial_lifecycle_hooks" { description = "Whether to create initial lifecycle hooks provided in worker groups." type = bool diff --git a/workers.tf b/workers.tf index 839452ed43..8d20bd8e12 100644 --- a/workers.tf +++ b/workers.tf @@ -220,7 +220,7 @@ resource "random_pet" "workers" { } resource "aws_security_group" "workers" { - count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 name_prefix = aws_eks_cluster.this[0].name description = "Security group for all nodes in the cluster." vpc_id = var.vpc_id @@ -234,7 +234,7 @@ resource "aws_security_group" "workers" { } resource "aws_security_group_rule" "workers_egress_internet" { - count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow nodes all egress to the Internet." protocol = "-1" security_group_id = local.worker_security_group_id @@ -245,7 +245,7 @@ resource "aws_security_group_rule" "workers_egress_internet" { } resource "aws_security_group_rule" "workers_ingress_self" { - count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow node to communicate with each other." protocol = "-1" security_group_id = local.worker_security_group_id @@ -256,7 +256,7 @@ resource "aws_security_group_rule" "workers_ingress_self" { } resource "aws_security_group_rule" "workers_ingress_cluster" { - count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow workers pods to receive communication from the cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id @@ -267,7 +267,7 @@ resource "aws_security_group_rule" "workers_ingress_cluster" { } resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { - count = var.worker_security_group_id == "" && var.create_eks ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 + count = var.worker_create_security_group && var.create_eks ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 description = "Allow workers Kubelets to receive communication from the cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id @@ -278,7 +278,7 @@ resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { } resource "aws_security_group_rule" "workers_ingress_cluster_https" { - count = var.worker_security_group_id == "" && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id From 3e4483517269f26591e3d54ea86233a5b5e97268 Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski Date: Thu, 27 Feb 2020 11:31:20 +0100 Subject: [PATCH 139/962] change default value for wait_for_cluster_cmd (#750) * change wait_for_cluster_cmd method change default method for wait_for_cluster_cmd from curl to wget to solve https://github.com/terraform-aws-modules/terraform-aws-eks/issues/741 * Update CHANGELOG.md * update docs Co-authored-by: Max Williams --- CHANGELOG.md | 1 + README.md | 2 +- variables.tf | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26ff3f3a1e..82479d4835 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - **Breaking:** Removal of autoscaling IAM policy and tags (by @max-rocket-internet) - Add `iam:{Create,Delete,Get}OpenIDConnectProvider` grants to the list of required IAM permissions in `docs/iam-permissions.md` (by @danielelisi) - Add an `name` parameter to be able to manually name EKS Managed Node Groups (by @splieth) +- Change variable default `wait_for_cluster_cmd` from curl to wget (by @daroga0002) #### Important notes diff --git a/README.md b/README.md index 97ac363496..3f4b265cb9 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | -| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"until curl -k -s $ENDPOINT/healthz \u003e/dev/null; do sleep 4; done"` | no | +| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"until wget --no-check-certificate -O - -q $ENDPOINT/healthz \u003e/dev/null; do sleep 4; done"` | no | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | | worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | diff --git a/variables.tf b/variables.tf index 9ba78b441a..b53766d912 100644 --- a/variables.tf +++ b/variables.tf @@ -201,7 +201,7 @@ variable "cluster_delete_timeout" { variable "wait_for_cluster_cmd" { description = "Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT" type = string - default = "until curl -k -s $ENDPOINT/healthz >/dev/null; do sleep 4; done" + default = "until wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null; do sleep 4; done" } variable "cluster_create_security_group" { From dda244994df974b7769dd078c060680161d36966 Mon Sep 17 00:00:00 2001 From: Andres De Castro Date: Thu, 27 Feb 2020 05:36:32 -0500 Subject: [PATCH 140/962] Pin version for kubernetes provider across examples to exactly 1.10.0 (#735) * Update README.md * Update main.tf * Update main.tf * Update main.tf * Update main.tf * Update main.tf * Update README.md * Update README.md * Update CHANGELOG.md Co-authored-by: Max Williams --- CHANGELOG.md | 1 + README.md | 2 +- examples/basic/main.tf | 2 +- examples/irsa/main.tf | 2 +- examples/launch_templates/main.tf | 2 +- examples/managed_node_groups/main.tf | 2 +- examples/spot_instances/main.tf | 2 +- 7 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82479d4835..8f05a81033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - **Breaking:** Removal of autoscaling IAM policy and tags (by @max-rocket-internet) - Add `iam:{Create,Delete,Get}OpenIDConnectProvider` grants to the list of required IAM permissions in `docs/iam-permissions.md` (by @danielelisi) - Add an `name` parameter to be able to manually name EKS Managed Node Groups (by @splieth) +- Pinned kubernetes provider version to exactly 1.10.0 across all examples and README.md's (by @andres-de-castro) - Change variable default `wait_for_cluster_cmd` from curl to wget (by @daroga0002) #### Important notes diff --git a/README.md b/README.md index 3f4b265cb9..f87546259d 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(element(concat(data.aws_eks_cluster.cluster[*].certificate_authority.0.data, list("")), 0)) token = element(concat(data.aws_eks_cluster_auth.cluster[*].token, list("")), 0) load_config_file = false - version = "~> 1.10" + version = "1.10" } # This cluster will not be created diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 05b36f6a48..6a89c3fa41 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -36,7 +36,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.10" + version = "1.10" } data "aws_availability_zones" "available" { diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index 2121a589ec..61cc54d6b3 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -32,7 +32,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.10" + version = "1.10" } data "aws_availability_zones" "available" {} diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 1c95a9fd83..534f2078bd 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -36,7 +36,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.10" + version = "1.10" } data "aws_availability_zones" "available" { diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index c31abb36b5..c2b8c236f0 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -36,7 +36,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.10" + version = "1.10" } data "aws_availability_zones" "available" { diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 8382d14c90..0dbb23e571 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -36,7 +36,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.10" + version = "1.10" } data "aws_availability_zones" "available" { From da53b5cab317f67b3f10e42c3bfd09993a2d9877 Mon Sep 17 00:00:00 2001 From: Igor Konforti Date: Thu, 27 Feb 2020 11:41:41 +0100 Subject: [PATCH 141/962] changing timeout (#736) * change timeout * Update README.md Co-authored-by: Max Williams --- CHANGELOG.md | 1 + README.md | 2 +- variables.tf | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f05a81033..8827deaf0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...HEAD)] - 2020-xx-xx] - Revert #631. Add back manage security group flags. (by @ryanooi) +- changed timeout for creating EKS (by @confiq) - Added instructions for how to add Windows nodes (by @ivanguravel) - [CI] Switch `Validate` github action to use env vars (by @max-rocket-internet) - [CI] Bump pre-commit-terraform version (by @barryib) diff --git a/README.md b/README.md index f87546259d..1c0a958f0e 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a |------|-------------|------|---------|:-----:| | attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | | cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | -| cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"15m"` | no | +| cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | | cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | diff --git a/variables.tf b/variables.tf index b53766d912..eac18e5eeb 100644 --- a/variables.tf +++ b/variables.tf @@ -189,7 +189,7 @@ variable "kubeconfig_name" { variable "cluster_create_timeout" { description = "Timeout value when creating the EKS cluster." type = string - default = "15m" + default = "30m" } variable "cluster_delete_timeout" { From 908c656cf5da50d576d77696efac1d76dfb86a07 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Thu, 27 Feb 2020 13:26:13 +0100 Subject: [PATCH 142/962] New release: v9.0.0 (#752) * New release: v9.0.0 * Resolve comments from barryib --- CHANGELOG.md | 24 ++++++++++++------------ version | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8827deaf0f..5bf92ef9e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,25 +7,29 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v8.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...HEAD)] - 2020-xx-xx] -- Revert #631. Add back manage security group flags. (by @ryanooi) +## [[v9.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...HEAD)] - 2020-xx-xx] + +- Write your awesome change here (by @you) + +# History -- changed timeout for creating EKS (by @confiq) +## [[v9.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...v9.0.0)] - 2020-02-27] + +- **Breaking:** Removal of autoscaling IAM policy and tags (by @max-rocket-internet) +- Revert #631. Add back manage security group flags. (by @ryanooi) +- Changed timeout for creating EKS (by @confiq) - Added instructions for how to add Windows nodes (by @ivanguravel) - [CI] Switch `Validate` github action to use env vars (by @max-rocket-internet) - [CI] Bump pre-commit-terraform version (by @barryib) - Added example `examples/irsa` for IAM Roles for Service Accounts (by @max-rocket-internet) -- **Breaking:** Removal of autoscaling IAM policy and tags (by @max-rocket-internet) - Add `iam:{Create,Delete,Get}OpenIDConnectProvider` grants to the list of required IAM permissions in `docs/iam-permissions.md` (by @danielelisi) -- Add an `name` parameter to be able to manually name EKS Managed Node Groups (by @splieth) +- Add a `name` parameter to be able to manually name EKS Managed Node Groups (by @splieth) - Pinned kubernetes provider version to exactly 1.10.0 across all examples and README.md's (by @andres-de-castro) - Change variable default `wait_for_cluster_cmd` from curl to wget (by @daroga0002) #### Important notes -Autoscaling policy and tags have been removed from this module. This reduces complexity and increases security as the policy was attached to the node group IAM role. To manage it outside of this module either follow the example in `examples/irsa` to attach an IAM role to the cluster-autoscaler `serviceAccount` or create the policy outside this module and pass it in using the `workers_additional_policies` variable. - -# History +Autoscaling policy and tags have been removed from this module. This reduces complexity and increases security as the policy was attached to the node group IAM role. To manage it outside of this module either follow the example in `examples/irsa` to attach an IAM role to the cluster-autoscaler `serviceAccount` or create [the policy](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/v8.2.0/workers.tf#L361-L416) outside this module and pass it in using the `workers_additional_policies` variable. ## [[v8.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...v8.2.0)] - 2020-01-29] @@ -33,8 +37,6 @@ Autoscaling policy and tags have been removed from this module. This reduces com - Disable creation of ingress rules if worker nodes security groups are exists (@andjelx) - [CI] Update pre-commit and re-generate docs to work with terraform-docs >= 0.8.1 (@barryib) -# History - ## [[v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0)] - 2020-01-17] - Fix index reference on destroy for output `oidc_provider_arn` (@stevie-) @@ -82,8 +84,6 @@ For security group whitelisting change. After upgrade, have to remove `cluster_c terraform import module.eks.aws_security_group_rule.cluster_https_worker_ingress _ingress_tcp_443_443_ ``` -# History - ## [[v7.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.1...v7.0.0)] - 2019-12-11] - Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) diff --git a/version b/version index 7c330f2dc9..40ed13353f 100644 --- a/version +++ b/version @@ -1 +1 @@ -v8.2.0 +v9.0.0 From 81ded922c811ca4b1f1ca299d083b367698e7c69 Mon Sep 17 00:00:00 2001 From: SW Date: Wed, 4 Mar 2020 15:59:58 +0100 Subject: [PATCH 143/962] cluster-autoscaler shouldn't be scheduled on spot (#755) * cluster-autoscaler shouldn't be scheduled on spot Update documentation to schedule cluster-autoscaler on normal instances. * Update CHANGELOG.md --- CHANGELOG.md | 2 +- docs/spot-instances.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bf92ef9e7..5268625b78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v9.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...HEAD)] - 2020-xx-xx] -- Write your awesome change here (by @you) +- Fix doc about spot instances, cluster-autoscaler should be scheduled on normal instances instead of spot (by @simowaer) # History diff --git a/docs/spot-instances.md b/docs/spot-instances.md index 1a5da6e79e..e8a31e45ab 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -12,7 +12,7 @@ In the following examples at least 1 worker group that uses on-demand instances ```yaml nodeSelector: - kubernetes.io/lifecycle: spot + kubernetes.io/lifecycle: normal ``` Notes: From 276e41f7440e1811f3fa44ebe3ee3e13cf89cac1 Mon Sep 17 00:00:00 2001 From: Craig Rueda Date: Fri, 6 Mar 2020 03:54:12 -0800 Subject: [PATCH 144/962] Adding encrypted option to root_block_device of launch_configuration (#761) * Adding encrypted option to root_block_device of launch_configuration * Updating CHANGELOG.md * Setting back to empty string --- CHANGELOG.md | 1 + workers.tf | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5268625b78..fb88268f50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v9.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...HEAD)] - 2020-xx-xx] - Fix doc about spot instances, cluster-autoscaler should be scheduled on normal instances instead of spot (by @simowaer) +- Adding `encrypted` option to worker's root_block_device as read from the worker configurations (by @craig-rueda) # History diff --git a/workers.tf b/workers.tf index 8d20bd8e12..4c1a6a05f4 100644 --- a/workers.tf +++ b/workers.tf @@ -185,6 +185,11 @@ resource "aws_launch_configuration" "workers" { ) root_block_device { + encrypted = lookup( + var.worker_groups[count.index], + "root_encrypted", + local.workers_group_defaults["root_encrypted"], + ) volume_size = lookup( var.worker_groups[count.index], "root_volume_size", From 8382f2c4e81cbbdc97f76bee828b855fd0d34787 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Mon, 9 Mar 2020 23:02:53 +0800 Subject: [PATCH 145/962] use the correct policy arns if the region is in CN (#765) --- CHANGELOG.md | 1 + cluster.tf | 4 ++-- data.tf | 2 ++ local.tf | 1 + workers.tf | 6 +++--- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb88268f50..a248bba146 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fix doc about spot instances, cluster-autoscaler should be scheduled on normal instances instead of spot (by @simowaer) - Adding `encrypted` option to worker's root_block_device as read from the worker configurations (by @craig-rueda) +- Use correct policy arns for CN regions (cn-north-1, cn-northwest-1) (by @cofyc) # History diff --git a/cluster.tf b/cluster.tf index c5439adea7..a327e920b6 100644 --- a/cluster.tf +++ b/cluster.tf @@ -96,12 +96,12 @@ resource "aws_iam_role" "cluster" { resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" { count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 - policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" + policy_arn = "${local.policy_arn_prefix}/AmazonEKSClusterPolicy" role = local.cluster_iam_role_name } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 - policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" + policy_arn = "${local.policy_arn_prefix}/AmazonEKSServicePolicy" role = local.cluster_iam_role_name } diff --git a/data.tf b/data.tf index f43ac9e680..d945c52833 100644 --- a/data.tf +++ b/data.tf @@ -221,3 +221,5 @@ data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instanc local.workers_group_defaults["iam_instance_profile_name"], ) } + +data "aws_region" "current" {} diff --git a/local.tf b/local.tf index 8ed32f8541..a959cc3360 100644 --- a/local.tf +++ b/local.tf @@ -22,6 +22,7 @@ locals { default_ami_id_linux = data.aws_ami.eks_worker.id default_ami_id_windows = data.aws_ami.eks_worker_windows.id + policy_arn_prefix = contains(["cn-northwest-1", "cn-north-1"], data.aws_region.current.name) ? "arn:aws-cn:iam::aws:policy" : "arn:aws:iam::aws:policy" workers_group_defaults_defaults = { name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. diff --git a/workers.tf b/workers.tf index 4c1a6a05f4..c198c5bca4 100644 --- a/workers.tf +++ b/workers.tf @@ -318,19 +318,19 @@ resource "aws_iam_instance_profile" "workers" { resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" + policy_arn = "${local.policy_arn_prefix}/AmazonEKSWorkerNodePolicy" role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { count = var.manage_worker_iam_resources && var.attach_worker_cni_policy && var.create_eks ? 1 : 0 - policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" + policy_arn = "${local.policy_arn_prefix}/AmazonEKS_CNI_Policy" role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" { count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + policy_arn = "${local.policy_arn_prefix}/AmazonEC2ContainerRegistryReadOnly" role = aws_iam_role.workers[0].name } From 6784824595ee4fba0660e049dc1cdfa564d83976 Mon Sep 17 00:00:00 2001 From: Craig Rueda Date: Mon, 9 Mar 2020 15:25:05 -0700 Subject: [PATCH 146/962] 'root_encrypted' -> bool (#766) --- local.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local.tf b/local.tf index a959cc3360..2151439569 100644 --- a/local.tf +++ b/local.tf @@ -67,7 +67,7 @@ locals { launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group launch_template_placement_tenancy = "default" # The placement tenancy for instances launch_template_placement_group = "" # The name of the placement group into which to launch the instances, if any. - root_encrypted = "" # Whether the volume should be encrypted or not + root_encrypted = false # Whether the volume should be encrypted or not eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs market_type = null From 198294c0b6451f3908cbe0fce192071829588338 Mon Sep 17 00:00:00 2001 From: Siddarth Prakash <1428486+sidprak@users.noreply.github.com> Date: Mon, 9 Mar 2020 18:31:52 -0400 Subject: [PATCH 147/962] Add support for ASG max instance lifetime (#769) Co-authored-by: Thierno IB. BARRY --- CHANGELOG.md | 1 + local.tf | 1 + workers.tf | 5 +++++ workers_launch_template.tf | 5 +++++ 4 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a248bba146..f2ec8e8416 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Fix doc about spot instances, cluster-autoscaler should be scheduled on normal instances instead of spot (by @simowaer) - Adding `encrypted` option to worker's root_block_device as read from the worker configurations (by @craig-rueda) - Use correct policy arns for CN regions (cn-north-1, cn-northwest-1) (by @cofyc) +- Add support for ASG max instance lifetime (by @sidprak) # History diff --git a/local.tf b/local.tf index 2151439569..e4969f7408 100644 --- a/local.tf +++ b/local.tf @@ -79,6 +79,7 @@ locals { spot_allocation_strategy = "lowest-price" # Valid options are 'lowest-price' and 'capacity-optimized'. If 'lowest-price', the Auto Scaling group launches instances using the Spot pools with the lowest price, and evenly allocates your instances across the number of Spot pools. If 'capacity-optimized', the Auto Scaling group launches instances using Spot pools that are optimally chosen based on the available Spot capacity. spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price + max_instance_lifetime = 0 # Maximum number of seconds instances can run in the ASG. 0 is unlimited. } workers_group_defaults = merge( diff --git a/workers.tf b/workers.tf index c198c5bca4..a4b9a3c222 100644 --- a/workers.tf +++ b/workers.tf @@ -73,6 +73,11 @@ resource "aws_autoscaling_group" "workers" { "termination_policies", local.workers_group_defaults["termination_policies"] ) + max_instance_lifetime = lookup( + var.worker_groups_launch_template[count.index], + "max_instance_lifetime", + local.workers_group_defaults["max_instance_lifetime"], + ) dynamic "initial_lifecycle_hook" { for_each = var.worker_create_initial_lifecycle_hooks ? lookup(var.worker_groups[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) : [] diff --git a/workers_launch_template.tf b/workers_launch_template.tf index ba51baaf9d..e3eb2ef4e1 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -72,6 +72,11 @@ resource "aws_autoscaling_group" "workers_launch_template" { "termination_policies", local.workers_group_defaults["termination_policies"] ) + max_instance_lifetime = lookup( + var.worker_groups_launch_template[count.index], + "max_instance_lifetime", + local.workers_group_defaults["max_instance_lifetime"], + ) dynamic mixed_instances_policy { iterator = item From f465e55721e31390fc64650c7c7eea517e862234 Mon Sep 17 00:00:00 2001 From: Arie Lev <34907201+ArieLevs@users.noreply.github.com> Date: Tue, 10 Mar 2020 12:20:11 +0200 Subject: [PATCH 148/962] Add ASG cooldown and health_check_grace_period (#770) * Add default_cooldown/health_check oprions * Update CHANGELOG.md * Update CHANGELOG.md Co-Authored-By: Thierno IB. BARRY Co-authored-by: Thierno IB. BARRY --- CHANGELOG.md | 1 + local.tf | 2 ++ workers.tf | 10 ++++++++++ workers_launch_template.tf | 10 ++++++++++ 4 files changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2ec8e8416..983081f041 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Adding `encrypted` option to worker's root_block_device as read from the worker configurations (by @craig-rueda) - Use correct policy arns for CN regions (cn-north-1, cn-northwest-1) (by @cofyc) - Add support for ASG max instance lifetime (by @sidprak) +- Add `default_cooldown` and `health_check_grace_period` options to workers ASG (by @ArieLevs) # History diff --git a/local.tf b/local.tf index e4969f7408..11751767b7 100644 --- a/local.tf +++ b/local.tf @@ -33,6 +33,8 @@ locals { asg_force_delete = false # Enable forced deletion for the autoscaling group. asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. asg_recreate_on_change = false # Recreate the autoscaling group when the Launch Template or Launch Configuration change. + default_cooldown = null # The amount of time, in seconds, after a scaling activity completes before another scaling activity can start. + health_check_grace_period = null # Time in seconds after instance comes into service before checking health. instance_type = "m4.large" # Size of the workers instances. spot_price = "" # Cost of spot instance. placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". diff --git a/workers.tf b/workers.tf index a4b9a3c222..66a19515c2 100644 --- a/workers.tf +++ b/workers.tf @@ -78,6 +78,16 @@ resource "aws_autoscaling_group" "workers" { "max_instance_lifetime", local.workers_group_defaults["max_instance_lifetime"], ) + default_cooldown = lookup( + var.worker_groups[count.index], + "default_cooldown", + local.workers_group_defaults["default_cooldown"] + ) + health_check_grace_period = lookup( + var.worker_groups[count.index], + "health_check_grace_period", + local.workers_group_defaults["health_check_grace_period"] + ) dynamic "initial_lifecycle_hook" { for_each = var.worker_create_initial_lifecycle_hooks ? lookup(var.worker_groups[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) : [] diff --git a/workers_launch_template.tf b/workers_launch_template.tf index e3eb2ef4e1..52ec738588 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -77,6 +77,16 @@ resource "aws_autoscaling_group" "workers_launch_template" { "max_instance_lifetime", local.workers_group_defaults["max_instance_lifetime"], ) + default_cooldown = lookup( + var.worker_groups_launch_template[count.index], + "default_cooldown", + local.workers_group_defaults["default_cooldown"] + ) + health_check_grace_period = lookup( + var.worker_groups_launch_template[count.index], + "health_check_grace_period", + local.workers_group_defaults["health_check_grace_period"] + ) dynamic mixed_instances_policy { iterator = item From 2715b01e4e3979070af9faf7d9763e4b529a9121 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 10 Mar 2020 22:01:01 +0100 Subject: [PATCH 149/962] fix max_instance_lifetime for workers (#771) * fix max_instance_lifetime for workers * update changelog * update changelog --- CHANGELOG.md | 1 + workers.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 983081f041..13f79d2f24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Use correct policy arns for CN regions (cn-north-1, cn-northwest-1) (by @cofyc) - Add support for ASG max instance lifetime (by @sidprak) - Add `default_cooldown` and `health_check_grace_period` options to workers ASG (by @ArieLevs) +- Fix support for ASG max instance lifetime for workers (by @barryib) # History diff --git a/workers.tf b/workers.tf index 66a19515c2..ce2b27c601 100644 --- a/workers.tf +++ b/workers.tf @@ -74,7 +74,7 @@ resource "aws_autoscaling_group" "workers" { local.workers_group_defaults["termination_policies"] ) max_instance_lifetime = lookup( - var.worker_groups_launch_template[count.index], + var.worker_groups[count.index], "max_instance_lifetime", local.workers_group_defaults["max_instance_lifetime"], ) From 23c005e5ef6107a52af37e6c17e9d572ebc3d81d Mon Sep 17 00:00:00 2001 From: Wolodja Wentland Date: Wed, 11 Mar 2020 11:26:06 +0000 Subject: [PATCH 150/962] Add EKS Secret envelope encryption support (#772) This adds support for configuring EKS clusters that utilise envelope encryption for Secrets: - https://aws.amazon.com/about-aws/whats-new/2020/03/amazon-eks-adds-envelope-encryption-for-secrets-with-aws-kms/ - https://aws.amazon.com/blogs/containers/using-eks-encryption-provider-support-for-defense-in-depth/ --- CHANGELOG.md | 1 + README.md | 3 +- cluster.tf | 11 +++ examples/secrets_encryption/main.tf | 118 +++++++++++++++++++++++ examples/secrets_encryption/outputs.tf | 24 +++++ examples/secrets_encryption/variables.tf | 52 ++++++++++ variables.tf | 9 ++ versions.tf | 2 +- 8 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 examples/secrets_encryption/main.tf create mode 100644 examples/secrets_encryption/outputs.tf create mode 100644 examples/secrets_encryption/variables.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index 13f79d2f24..86cfb3dd72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Add support for ASG max instance lifetime (by @sidprak) - Add `default_cooldown` and `health_check_grace_period` options to workers ASG (by @ArieLevs) - Fix support for ASG max instance lifetime for workers (by @barryib) +- Add support for envelope encryption of Secrets (by @babilen5) # History diff --git a/README.md b/README.md index 1c0a958f0e..7b2ad00b04 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| aws | >= 2.44.0 | +| aws | >= 2.52.0 | | kubernetes | >= 1.6.2 | | local | >= 1.2 | | null | >= 2.1 | @@ -166,6 +166,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | | cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | +| cluster\_encryption\_config | Configuration block with encryption configuration for the cluster. See examples/secrets\_encryption/main.tf for example format |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | | cluster\_endpoint\_public\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | diff --git a/cluster.tf b/cluster.tf index a327e920b6..2e0ca6862a 100644 --- a/cluster.tf +++ b/cluster.tf @@ -27,6 +27,17 @@ resource "aws_eks_cluster" "this" { delete = var.cluster_delete_timeout } + dynamic encryption_config { + for_each = toset(var.cluster_encryption_config) + + content { + provider { + key_arn = encryption_config.value["provider_key_arn"] + } + resources = encryption_config.value["resources"] + } + } + depends_on = [ aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy, aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy, diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf new file mode 100644 index 0000000000..67057e24b8 --- /dev/null +++ b/examples/secrets_encryption/main.tf @@ -0,0 +1,118 @@ +terraform { + required_version = ">= 0.12.0" +} + +provider "aws" { + version = ">= 2.52.0" + region = var.region +} + +provider "random" { + version = "~> 2.1" +} + +provider "local" { + version = "~> 1.2" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false + version = "1.10" +} + +data "aws_availability_zones" "available" { +} + +locals { + cluster_name = "test-eks-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +resource "aws_kms_key" "eks" { + description = "EKS Secret Encryption Key" +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.6.0" + + name = "test-vpc" + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } +} + +module "eks" { + source = "../.." + cluster_name = local.cluster_name + subnets = module.vpc.private_subnets + + cluster_encryption_config = [ + { + provider_key_arn = aws_kms_key.eks.arn + resources = ["secrets"] + } + ] + + tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + vpc_id = module.vpc.vpc_id + + worker_groups = [ + { + name = "worker-group-1" + instance_type = "t2.small" + additional_userdata = "echo foo bar" + asg_desired_capacity = 2 + }, + ] + + map_roles = var.map_roles + map_users = var.map_users + map_accounts = var.map_accounts +} diff --git a/examples/secrets_encryption/outputs.tf b/examples/secrets_encryption/outputs.tf new file mode 100644 index 0000000000..51ddb024a2 --- /dev/null +++ b/examples/secrets_encryption/outputs.tf @@ -0,0 +1,24 @@ +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = module.eks.cluster_security_group_id +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = module.eks.kubeconfig +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = module.eks.config_map_aws_auth +} + +output "region" { + description = "AWS region." + value = var.region +} diff --git a/examples/secrets_encryption/variables.tf b/examples/secrets_encryption/variables.tf new file mode 100644 index 0000000000..7085aeabd4 --- /dev/null +++ b/examples/secrets_encryption/variables.tf @@ -0,0 +1,52 @@ +variable "region" { + default = "us-west-2" +} + +variable "map_accounts" { + description = "Additional AWS account numbers to add to the aws-auth configmap." + type = list(string) + + default = [ + "777777777777", + "888888888888", + ] +} + +variable "map_roles" { + description = "Additional IAM roles to add to the aws-auth configmap." + type = list(object({ + rolearn = string + username = string + groups = list(string) + })) + + default = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] +} + +variable "map_users" { + description = "Additional IAM users to add to the aws-auth configmap." + type = list(object({ + userarn = string + username = string + groups = list(string) + })) + + default = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] +} diff --git a/variables.tf b/variables.tf index eac18e5eeb..0ba56ac58c 100644 --- a/variables.tf +++ b/variables.tf @@ -311,3 +311,12 @@ variable "eks_oidc_root_ca_thumbprint" { description = "Thumbprint of Root CA for EKS OIDC, Valid until 2037" default = "9e99a48a9960b14926bb7f3b02e22da2b0ab7280" } + +variable "cluster_encryption_config" { + description = "Configuration block with encryption configuration for the cluster. See examples/secrets_encryption/main.tf for example format" + type = list(object({ + provider_key_arn = string + resources = list(string) + })) + default = [] +} diff --git a/versions.tf b/versions.tf index 95fb1ef19e..f73abd331e 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.12.9" required_providers { - aws = ">= 2.44.0" + aws = ">= 2.52.0" local = ">= 1.2" null = ">= 2.1" template = ">= 2.1" From 7dc56e976bf0f8c48820085a4cdee37e676ca2b8 Mon Sep 17 00:00:00 2001 From: Scott Crooks Date: Wed, 11 Mar 2020 17:06:55 +0100 Subject: [PATCH 151/962] EKS 1.15 Support (#776) * chore(eks): Support EKS 1.15 * docs(changelog): Adding entry to CHANGELOG * chore(examples): Removing VPC tags from merged PR #772 * docs(changelog): Adding 'breaking' to CHANGELOG entry --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 1 + README.md | 2 +- examples/basic/main.tf | 4 ---- examples/irsa/main.tf | 4 ---- examples/launch_templates/main.tf | 4 ---- examples/managed_node_groups/main.tf | 4 ---- examples/secrets_encryption/main.tf | 4 ---- examples/spot_instances/main.tf | 4 ---- variables.tf | 2 +- 10 files changed, 4 insertions(+), 27 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b77bc2d98..71ccc545ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.25.0 + rev: v1.27.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/CHANGELOG.md b/CHANGELOG.md index 86cfb3dd72..a3fe093129 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Add `default_cooldown` and `health_check_grace_period` options to workers ASG (by @ArieLevs) - Fix support for ASG max instance lifetime for workers (by @barryib) - Add support for envelope encryption of Secrets (by @babilen5) +- **Breaking:** Added support for EKS 1.15 (by @sc250024) # History diff --git a/README.md b/README.md index 7b2ad00b04..b370a2e3d6 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | -| cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | `"1.14"` | no | +| cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | `"1.15"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | | create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | eks\_oidc\_root\_ca\_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 6a89c3fa41..ee1d9bc251 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -111,10 +111,6 @@ module "vpc" { single_nat_gateway = true enable_dns_hostnames = true - tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - } - public_subnet_tags = { "kubernetes.io/cluster/${local.cluster_name}" = "shared" "kubernetes.io/role/elb" = "1" diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index 61cc54d6b3..0fd934d44f 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -48,10 +48,6 @@ module "vpc" { public_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] enable_dns_hostnames = true - tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - } - public_subnet_tags = { "kubernetes.io/cluster/${local.cluster_name}" = "shared" "kubernetes.io/role/elb" = "1" diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 534f2078bd..a8d0ec62fe 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -60,10 +60,6 @@ module "vpc" { azs = data.aws_availability_zones.available.names public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] enable_dns_hostnames = true - - tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - } } module "eks" { diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index c2b8c236f0..adb5cfe70c 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -64,10 +64,6 @@ module "vpc" { single_nat_gateway = true enable_dns_hostnames = true - tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - } - public_subnet_tags = { "kubernetes.io/cluster/${local.cluster_name}" = "shared" "kubernetes.io/role/elb" = "1" diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index 67057e24b8..5c4b861be0 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -68,10 +68,6 @@ module "vpc" { single_nat_gateway = true enable_dns_hostnames = true - tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - } - public_subnet_tags = { "kubernetes.io/cluster/${local.cluster_name}" = "shared" "kubernetes.io/role/elb" = "1" diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 0dbb23e571..cfac972f57 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -60,10 +60,6 @@ module "vpc" { azs = data.aws_availability_zones.available.names public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] enable_dns_hostnames = true - - tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - } } module "eks" { diff --git a/variables.tf b/variables.tf index 0ba56ac58c..925b7b3109 100644 --- a/variables.tf +++ b/variables.tf @@ -28,7 +28,7 @@ variable "cluster_security_group_id" { variable "cluster_version" { description = "Kubernetes version to use for the EKS cluster." type = string - default = "1.14" + default = "1.15" } variable "config_output_path" { From 2a8e45fe2c54f0bc6a4fc68b5b61930ad22943f9 Mon Sep 17 00:00:00 2001 From: Christian Kemper Date: Wed, 11 Mar 2020 23:34:21 +0000 Subject: [PATCH 152/962] defaults lookup before falling back on default ami based on platform (#756) * defaulting to data lookup if worker_group_defaults have no ami_id entry * using coalesce instead of lookup and also using local instead of var. * adding defaults support for specifying windows based amis --- CHANGELOG.md | 1 + local.tf | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3fe093129..e333fa6661 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v9.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...HEAD)] - 2020-xx-xx] +- Ensuring that ami lookup hierarchy is worker_group_launch_templates and worker_groups -> worker_group_defaults -> and finally aws ami lookup (by @ck3mp3r) - Fix doc about spot instances, cluster-autoscaler should be scheduled on normal instances instead of spot (by @simowaer) - Adding `encrypted` option to worker's root_block_device as read from the worker configurations (by @craig-rueda) - Use correct policy arns for CN regions (cn-north-1, cn-northwest-1) (by @cofyc) diff --git a/local.tf b/local.tf index 11751767b7..704b5cdcb2 100644 --- a/local.tf +++ b/local.tf @@ -19,14 +19,15 @@ locals { worker_group_count = length(var.worker_groups) worker_group_launch_template_count = length(var.worker_groups_launch_template) - default_ami_id_linux = data.aws_ami.eks_worker.id - default_ami_id_windows = data.aws_ami.eks_worker_windows.id + default_ami_id_linux = coalesce(local.workers_group_defaults.ami_id, data.aws_ami.eks_worker.id) + default_ami_id_windows = coalesce(local.workers_group_defaults.ami_id_windows, data.aws_ami.eks_worker_windows.id) policy_arn_prefix = contains(["cn-northwest-1", "cn-north-1"], data.aws_region.current.name) ? "arn:aws-cn:iam::aws:policy" : "arn:aws:iam::aws:policy" workers_group_defaults_defaults = { name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. - ami_id = "" # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. + ami_id = "" # AMI ID for the eks linux based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. + ami_id_windows = "" # AMI ID for the eks windows based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. asg_max_size = "3" # Maximum worker capacity in the autoscaling group. asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. From 2a78825ef79753e6ea78687bfedd8e9b684187bd Mon Sep 17 00:00:00 2001 From: pierresteiner <32190738+pierresteiner@users.noreply.github.com> Date: Thu, 12 Mar 2020 10:34:13 +0100 Subject: [PATCH 153/962] Remove assumption (#779) * Remove assumption This is not needed since 8.0.0 right? * Update CHANGELOG.md Co-authored-by: Thierno IB. BARRY --- CHANGELOG.md | 1 + README.md | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e333fa6661..19b346234c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v9.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...HEAD)] - 2020-xx-xx] +- Remove obsolete assumption from README (kubectl & aws-iam-authenticator) (by @pierresteiner) - Ensuring that ami lookup hierarchy is worker_group_launch_templates and worker_groups -> worker_group_defaults -> and finally aws ami lookup (by @ck3mp3r) - Fix doc about spot instances, cluster-autoscaler should be scheduled on normal instances instead of spot (by @simowaer) - Adding `encrypted` option to worker's root_block_device as read from the worker configurations (by @craig-rueda) diff --git a/README.md b/README.md index b370a2e3d6..a21694b5ce 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,6 @@ Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aw * You want to create an EKS cluster and an autoscaling group of workers for the cluster. * You want these resources to exist within security groups that allow communication and coordination. These can be user provided or created within the module. * You've created a Virtual Private Cloud (VPC) and subnets where you intend to put the EKS resources. The VPC satisfies [EKS requirements](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html). -* If `manage_aws_auth = true`, it's required that both [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) (>=1.10) and [`aws-iam-authenticator`](https://github.com/kubernetes-sigs/aws-iam-authenticator#4-set-up-kubectl-to-use-authentication-tokens-provided-by-aws-iam-authenticator-for-kubernetes) are installed and on your shell's PATH. ## Usage example From 4c0c4c49a611e0d7dcaff334fc7862c31732d1f0 Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski Date: Thu, 12 Mar 2020 11:50:41 +0100 Subject: [PATCH 154/962] change conditional for irsa.tf (#783) --- CHANGELOG.md | 1 + irsa.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19b346234c..5388e9878d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Add `default_cooldown` and `health_check_grace_period` options to workers ASG (by @ArieLevs) - Fix support for ASG max instance lifetime for workers (by @barryib) - Add support for envelope encryption of Secrets (by @babilen5) +- Fix issue with terraform plan phase when IRSA was enabled and create_eks switches to false (by @daroga0002) - **Breaking:** Added support for EKS 1.15 (by @sc250024) # History diff --git a/irsa.tf b/irsa.tf index 08096d8f08..11c0cc735c 100644 --- a/irsa.tf +++ b/irsa.tf @@ -8,7 +8,7 @@ # https://github.com/terraform-providers/terraform-provider-aws/issues/10104 resource "aws_iam_openid_connect_provider" "oidc_provider" { - count = var.enable_irsa ? 1 : 0 + count = var.enable_irsa && var.create_eks ? 1 : 0 client_id_list = ["sts.amazonaws.com"] thumbprint_list = [var.eks_oidc_root_ca_thumbprint] url = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0] From ca3d1e1199b2e19c67c7c36fcb1f60c010575208 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 12 Mar 2020 13:29:03 +0100 Subject: [PATCH 155/962] New release 10.0.0 (#780) * prepare 10.0.0 release * remove unused version file --- CHANGELOG.md | 98 ++++++++++++++++++++++++++++++---------------------- version | 1 - 2 files changed, 57 insertions(+), 42 deletions(-) delete mode 100644 version diff --git a/CHANGELOG.md b/CHANGELOG.md index 5388e9878d..343031df6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,23 +7,39 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Next release -## [[v9.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...HEAD)] - 2020-xx-xx] +## [[v10.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v10.0.0...HEAD)] - 2020-xx-xx + +- Write your awesome change here (GH-xxxx) + +## History + +## [[v10.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...v10.0.0)] - 2020-03-12 + +BREAKING CHANGES: + +- Added support for EKS 1.15 (by @sc250024) + +ENHANCEMENTS: -- Remove obsolete assumption from README (kubectl & aws-iam-authenticator) (by @pierresteiner) - Ensuring that ami lookup hierarchy is worker_group_launch_templates and worker_groups -> worker_group_defaults -> and finally aws ami lookup (by @ck3mp3r) -- Fix doc about spot instances, cluster-autoscaler should be scheduled on normal instances instead of spot (by @simowaer) - Adding `encrypted` option to worker's root_block_device as read from the worker configurations (by @craig-rueda) -- Use correct policy arns for CN regions (cn-north-1, cn-northwest-1) (by @cofyc) - Add support for ASG max instance lifetime (by @sidprak) - Add `default_cooldown` and `health_check_grace_period` options to workers ASG (by @ArieLevs) -- Fix support for ASG max instance lifetime for workers (by @barryib) - Add support for envelope encryption of Secrets (by @babilen5) + +BUG FIXES: + - Fix issue with terraform plan phase when IRSA was enabled and create_eks switches to false (by @daroga0002) -- **Breaking:** Added support for EKS 1.15 (by @sc250024) +- Remove obsolete assumption from README (kubectl & aws-iam-authenticator) (by @pierresteiner) +- Fix doc about spot instances, cluster-autoscaler should be scheduled on normal instances instead of spot (by @simowaer) +- Use correct policy arns for CN regions (cn-north-1, cn-northwest-1) (by @cofyc) +- Fix support for ASG max instance lifetime for workers (by @barryib) + +NOTES: -# History +From EKS 1.15, the VPC tag `kubernetes.io/cluster/: shared` is no longer required. So we droped those tags from exemples. -## [[v9.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...v9.0.0)] - 2020-02-27] +## [[v9.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...v9.0.0)] - 2020-02-27 - **Breaking:** Removal of autoscaling IAM policy and tags (by @max-rocket-internet) - Revert #631. Add back manage security group flags. (by @ryanooi) @@ -41,19 +57,19 @@ project adheres to [Semantic Versioning](http://semver.org/). Autoscaling policy and tags have been removed from this module. This reduces complexity and increases security as the policy was attached to the node group IAM role. To manage it outside of this module either follow the example in `examples/irsa` to attach an IAM role to the cluster-autoscaler `serviceAccount` or create [the policy](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/v8.2.0/workers.tf#L361-L416) outside this module and pass it in using the `workers_additional_policies` variable. -## [[v8.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...v8.2.0)] - 2020-01-29] +## [[v8.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...v8.2.0)] - 2020-01-29 - Include ability to configure custom os-specific command for waiting until kube cluster is healthy (@sanjeevgiri) - Disable creation of ingress rules if worker nodes security groups are exists (@andjelx) - [CI] Update pre-commit and re-generate docs to work with terraform-docs >= 0.8.1 (@barryib) -## [[v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0)] - 2020-01-17] +## [[v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0)] - 2020-01-17 - Fix index reference on destroy for output `oidc_provider_arn` (@stevie-) - Add support for restricting access to the public API endpoint (@sidprak) - Add an `ignore_lifecycle` rule to prevent Terraform from scaling down ASG behind AWS EKS Managed Node Group (by @davidalger) -## [[v8.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v7.0.1)] - 2020-01-09] +## [[v8.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v7.0.1)] - 2020-01-09 - **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) - **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. See Important notes below for upgrade notes (by @sdehaes) @@ -94,14 +110,14 @@ For security group whitelisting change. After upgrade, have to remove `cluster_c terraform import module.eks.aws_security_group_rule.cluster_https_worker_ingress _ingress_tcp_443_443_ ``` -## [[v7.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.1...v7.0.0)] - 2019-12-11] +## [[v7.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.1...v7.0.0)] - 2019-12-11 - Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) - Updated `instance_profile_names` and `instance_profile_arns` outputs to also consider launch template as well as asg (by @ankitwal) - Fix broken terraform plan/apply on a cluster < 1.14 (by @hodduc) - Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) -## [[v7.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v7.0.0)] - 2019-10-30] +## [[v7.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v7.0.0)] - 2019-10-30 - **Breaking:** Allow for specifying a custom AMI for the worker nodes. (by @bmcstdio) - Added support for Windows workers AMIs (by @hodduc) @@ -115,20 +131,20 @@ terraform import module.eks.aws_security_group_rule.cluster_https_worker_ingress An AMI is now specified using the whole name, for example `amazon-eks-node-1.14-v20190927`. -## [[v6.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2)] - 2019-10-07] +## [[v6.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2)] - 2019-10-07 - Added `tags` to `aws_eks_cluster` introduced by terraform-provider-aws 2.31.0 (by @morganchristiansson) - Add option to enable lifecycle hooks creation (by @barryib) - Remove helm chart value `sslCertPath` described in `docs/autoscaling.md` (by @wi1dcard) - Attaching of IAM policies for autoscaler and CNI to the worker nodes now optional (by @dpiddockcmp) -## [[v6.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...v6.0.1)] - 2019-09-25] +## [[v6.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...v6.0.1)] - 2019-09-25 - Added support for different workers AMI's, i.e. with GPU support (by @rvoitenko) - Use null as default value for `target_group_arns` attribute of worker autoscaling group (by @tatusl) - Output empty string when cluster identity is empty (by @tbarry) -## [[v6.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v6.0.0)] - 2019-09-17] +## [[v6.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v6.0.0)] - 2019-09-17 - Added `market_type` to `workers_launch_template.tf` allow the usage of spot nodegroups without mixed instances policy. - Added support for log group tag in `./cluster.tf` (@lucas-giaco) @@ -156,11 +172,11 @@ You will need to move worker groups from `worker_groups_launch_template_mixed` t Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [""]`. -## [[v5.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v5.1.1)] - 2019-07-30] +## [[v5.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v5.1.1)] - 2019-07-30 - Added new tag in `worker.tf` with autoscaling_enabled = true flag (by @insider89) -## [[v5.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...v5.1.0)] - 2019-07-30] +## [[v5.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...v5.1.0)] - 2019-07-30 - Option to set a KMS key for the log group and encrypt it (by @till-krauss) - Output the name of the cloudwatch log group (by @gbooth27) @@ -170,7 +186,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Update default override instance types to work with Cluster Autoscaler (by @nauxliu on behalf of RightCapital) - Examples now specify `enable_dns_hostnames = true`, as per [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html) (by @karolinepauls) -## [[v5.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...v5.0.0)] - 2019-06-19] +## [[v5.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...v5.0.0)] - 2019-06-19 - Added Termination Policy Option to worker ASGs (by @undeadops) - Update EBS optimized instances type (by @gloutsch) @@ -184,17 +200,17 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Change override instance_types to list (by @nauxliu on behalf of RightCapital) - Fix toggle for IAM instance profile creation for mixed launch templates (by @jnozo) -## [[v4.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.1...v4.0.2)] - 2019-05-07] +## [[v4.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.1...v4.0.2)] - 2019-05-07 - Added 2 new examples, also tidy up basic example (by @max-rocket-internet) - Updates to travis, PR template (by @max-rocket-internet) - Fix typo in data.tf (by @max-rocket-internet) - Add missing launch template items in `aws_auth.tf` (by @max-rocket-internet) -## [[v4.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1)] - 2019-05-07] +## [[v4.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1)] - 2019-05-07 - Fix annoying typo: worker_group_xx vs worker_groups_xx (by @max-rocket-internet) -## [[v4.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v3.0.0...v4.0.0)] - 2019-05-07] +## [[v4.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v3.0.0...v4.0.0)] - 2019-05-07 - Added support for custom service linked role for Auto Scaling group (by @voanhduy1512) - Added support for custom IAM roles for cluster and workers (by @erks) @@ -209,7 +225,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Switch to https for the pre-commit repos (by @rothandrew) - Add instructions on how to enable the docker bridge network (by @rothandrew) -## [[v3.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.1...v3.0.0)] - 2019-04-15] +## [[v3.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.1...v3.0.0)] - 2019-04-15 - Fixed: Ability to destroy clusters due to security groups being attached to ENI's (by @whiskeyjimbo) - Added outputs for worker IAM instance profile(s) (by @soapergem) @@ -217,13 +233,13 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Updated vpc module version and aws provider version. (by @chenrui333) - Upgraded default kubernetes version from 1.11 to 1.12 (by @stijndehaes) -## [[v2.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.0...v2.3.1)] - 2019-03-26] +## [[v2.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.0...v2.3.1)] - 2019-03-26 - Added support for eks public and private endpoints (by @stijndehaes) - Added minimum inbound traffic rule to the cluster worker security group as per the [EKS security group requirements](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) (by @sc250024) - (Breaking Change) Replaced `enable_docker_bridge` with a generic option called `bootstrap_extra_args` to resolve [310](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/310) (by @max-rocket-internet) -## [[v2.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.3.0)] - 2019-03-20] +## [[v2.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.3.0)] - 2019-03-20 - Allow additional policies to be attached to worker nodes (by @rottenbytes) - Ability to specify a placement group for each worker group (by @matheuss) @@ -235,13 +251,13 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Added optional name filter variable to be able to pin worker AMI to a release (by @max-rocket-internet) - Added `--enable-docker-bridge` option for bootstrap.sh in AMI (by @michaelmccord) -## [[v2.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.2.2)] - 2019-02-25] +## [[v2.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.2.2)] - 2019-02-25 - Ability to specify a path for IAM roles (by @tekn0ir) -## [[v2.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.0...v2.2.1)] - 2019-02-18] +## [[v2.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.0...v2.2.1)] - 2019-02-18 -## [[v2.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.1.0...v2.2.0)] - 2019-02-07] +## [[v2.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.1.0...v2.2.0)] - 2019-02-07 - Ability to specify a permissions_boundary for IAM roles (by @dylanhellems) - Ability to configure force_delete for the worker group ASG (by @stefansedich) @@ -253,7 +269,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Fixed a bug where worker group defaults were being used for launch template user data (by @leonsodhi-lf) - Managed_aws_auth option is true, the aws-auth configmap file is no longer created, and write_aws_auth_config must be set to true to generate config_map. (by @yutachaos) -## [[v2.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.0.0...v2.1.0)] - 2019-01-15] +## [[v2.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.0.0...v2.1.0)] - 2019-01-15 - Initial support for worker groups based on Launch Templates (by @skang0601) - Updated the `update_config_map_aws_auth` resource to trigger when the EKS cluster endpoint changes. This likely means that a new cluster was spun up so our ConfigMap won't exist (fixes #234) (by @elatt) @@ -261,7 +277,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Fixed zsh-specific syntax in retry loop for aws auth config map (by @marcelloromani) - Fix: fail deployment if applying the aws auth config map still fails after 10 attempts (by @marcelloromani) -## [[v2.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.8.0...v2.0.0)] - 2018-12-14] +## [[v2.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.8.0...v2.0.0)] - 2018-12-14 - (Breaking Change) New input variables `map_accounts_count`, `map_roles_count` and `map_users_count` to allow using computed values as part of `map_accounts`, `map_roles` and `map_users` configs (by @chili-man on behalf of OpenGov). - (Breaking Change) New variables `cluster_create_security_group` and `worker_create_security_group` to stop `value of 'count' cannot be computed` error. @@ -272,7 +288,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Travis fixes (by @RothAndrew) - Fixed some Windows compatibility issues (by @RothAndrew) -## [[v1.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.7.0...v1.8.0)] - 2018-12-04] +## [[v1.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.7.0...v1.8.0)] - 2018-12-04 - Support for using AWS Launch Templates to define autoscaling groups (by @skang0601) - `suspended_processes` to `worker_groups` input (by @bkmeneguello) @@ -286,7 +302,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Removed providers from `main.tf`. (by @max-rocket-internet) - Removed `configure_kubectl_session` references in documentation [#171](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/171) (by @dominik-k) -## [[v1.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.6.0...v1.7.0)] - 2018-10-09] +## [[v1.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.6.0...v1.7.0)] - 2018-10-09 - Worker groups can be created with a specified IAM profile. (from @laverya) - exposed `aws_eks_cluster` create and destroy timeouts (by @RGPosadas) @@ -297,7 +313,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Cosmetic syntax changes to improve readability. (by @max-rocket-internet) - add `protect_from_scale_in` to solve issue #134 (by @kinghajj) -## [[v1.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.5.0...v1.6.0)] - 2018-09-04] +## [[v1.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.5.0...v1.6.0)] - 2018-09-04 - add support for [`amazon-eks-node-*` AMI with bootstrap script](https://aws.amazon.com/blogs/opensource/improvements-eks-worker-node-provisioning/) (by @erks) - expose `kubelet_extra_args` worker group option (replacing `kubelet_node_labels`) to allow specifying arbitrary kubelet options (e.g. taints and labels) (by @erks) @@ -308,7 +324,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - **Breaking change** Removed support for `eks-worker-*` AMI. The cluster specifying a custom AMI based off of `eks-worker-*` AMI will have to rebuild the AMI from `amazon-eks-node-*`. (by @erks) - **Breaking change** Removed `kubelet_node_labels` worker group option in favor of `kubelet_extra_args`. (by @erks) -## [[v1.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.4.0...v1.5.0)] - 2018-08-30] +## [[v1.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.4.0...v1.5.0)] - 2018-08-30 - add spot_price option to aws_launch_configuration - add enable_monitoring option to aws_launch_configuration @@ -319,20 +335,20 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ If you are upgrading from 1.4 you should fix state after upgrade: `terraform state rm module.eks.data.http.workstation_external_ip` - Can now selectively override keys in `workers_group_defaults` variable rather than callers maintaining a duplicate of the whole map. (by @dpiddockcmp) -## [[v1.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.3.0...v1.4.0)] - 2018-08-02] +## [[v1.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.3.0...v1.4.0)] - 2018-08-02 - manage eks workers' root volume size and type. - `workers_asg_names` added to outputs. (kudos to @laverya) - New top level variable `worker_group_count` added to replace the use of `length(var.worker_groups)`. This allows using computed values as part of worker group configs. (complaints to @laverya) -## [[v1.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.2.0...v1.3.0)] - 2018-07-11] +## [[v1.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.2.0...v1.3.0)] - 2018-07-11 - New variables `map_accounts`, `map_roles` and `map_users` in order to manage additional entries in the `aws-auth` configmap. (by @max-rocket-internet) - kubelet_node_labels worker group option allows setting --node-labels= in kubelet. (Hat-tip, @bshelton229 👒) - `worker_iam_role_arn` added to outputs. Sweet, @hatemosphere 🔥 - Worker subnets able to be specified as a dedicated list per autoscaling group. (up top, @bshelton229 🙏) -## [[v1.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.1.0...v1.2.0)] - 2018-07-01] +## [[v1.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.1.0...v1.2.0)] - 2018-07-01 - new variable `pre_userdata` added to worker launch configuration allows to run scripts before the plugin does anything. (W00t, @jimbeck 🦉) - kubeconfig made much more flexible. (Bang up job, @sdavids13 💥) @@ -340,7 +356,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Providing security groups didn't behave as expected. This has been fixed. (Good catch, @jimbeck 🔧) - workstation cidr to be allowed by created security group is now more flexible. (A welcome addition, @jimbeck 🔐) -## [[v1.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.0.0...v1.1.0)] - 2018-06-25] +## [[v1.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.0.0...v1.1.0)] - 2018-06-25 - new variable `worker_sg_ingress_from_port` allows to change the minimum port number from which pods will accept communication (Thanks, @ilyasotkov 👏). - expanded on worker example to show how multiple worker autoscaling groups can be created. @@ -351,13 +367,13 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - `kube-proxy` restart no longer needed in userdata. Good catch, @hatemosphere 🔥 - worker ASG reattachment wasn't possible when using `name`. Moved to `name_prefix` to allow recreation of resources. Kudos again, @hatemosphere 🐧 -## [[v1.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.2.0...v1.0.0)] - 2018-06-11] +## [[v1.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.2.0...v1.0.0)] - 2018-06-11 - security group id can be provided for either/both of the cluster and the workers. If not provided, security groups will be created with sufficient rules to allow cluster-worker communication. - kudos to @tanmng on the idea ⭐ - outputs of security group ids and worker ASG arns added for working with these resources outside the module. - Worker build out refactored to allow multiple autoscaling groups each having differing specs. If none are given, a single ASG is created with a set of sane defaults - big thanks to @kppullin 🥨 -## [[v0.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.1...v0.2.0)] - 2018-06-08] +## [[v0.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.1...v0.2.0)] - 2018-06-08 - ability to specify extra userdata code to execute following kubelet services start. - EBS optimization used whenever possible for the given instance type. @@ -365,7 +381,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - files rendered from dedicated templates to separate out raw code and config from `hcl` - `workers_ami_id` is now made optional. If not specified, the module will source the latest AWS supported EKS AMI instead. -## [[v0.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.0...v0.1.1)] - 2018-06-07] +## [[v0.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.0...v0.1.1)] - 2018-06-07 - Pre-commit hooks fixed and working. - Made progress on CI, advancing the build to the final `kitchen test` stage before failing. diff --git a/version b/version deleted file mode 100644 index 40ed13353f..0000000000 --- a/version +++ /dev/null @@ -1 +0,0 @@ -v9.0.0 From 9fb1a20d82daddd9da29002811640049e00d228c Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 17 Mar 2020 14:22:22 +0100 Subject: [PATCH 156/962] improvement: automate changelog management (#786) --- .chglog/CHANGELOG.tpl.md | 66 +++ .chglog/config.yml | 48 ++ .github/CONTRIBUTING.md | 34 ++ .github/PULL_REQUEST_TEMPLATE.md | 1 - .github/semantic.yml | 14 + .github/workflows/changelog-check.yaml | 20 + CHANGELOG.md => CHANGELOG.pre-v11.0.0.md | 78 ++- CONTRIBUTING.md | 13 - Makefile | 17 + README.md | 3 +- tools/semtag | 627 +++++++++++++++++++++++ 11 files changed, 863 insertions(+), 58 deletions(-) create mode 100644 .chglog/CHANGELOG.tpl.md create mode 100644 .chglog/config.yml create mode 100644 .github/CONTRIBUTING.md create mode 100644 .github/semantic.yml create mode 100644 .github/workflows/changelog-check.yaml rename CHANGELOG.md => CHANGELOG.pre-v11.0.0.md (85%) delete mode 100644 CONTRIBUTING.md create mode 100644 Makefile create mode 100755 tools/semtag diff --git a/.chglog/CHANGELOG.tpl.md b/.chglog/CHANGELOG.tpl.md new file mode 100644 index 0000000000..52d22241c0 --- /dev/null +++ b/.chglog/CHANGELOG.tpl.md @@ -0,0 +1,66 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) and this +project adheres to [Semantic Versioning](http://semver.org/). + +{{ if .Versions -}} + +## [Unreleased] +{{ if .Unreleased.CommitGroups -}} +{{ range .Unreleased.CommitGroups -}} +{{ .Title }}: +{{ range .Commits -}} +{{- if .Subject -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{- end -}} +{{ end }} +{{ end -}} +{{ else }} +{{ range .Unreleased.Commits -}} +{{- if .Subject -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{- end -}} +{{ end }} +{{ end -}} +{{ end -}} + +{{ range .Versions }} + +## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }} +{{ if .CommitGroups -}} +{{ range .CommitGroups -}} +{{ .Title }}: +{{ range .Commits -}} +{{- if .Subject -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{- end -}} +{{ end }} +{{ end -}} +{{ else }} +{{ range .Commits -}} +{{- if .Subject -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{- end -}} +{{ end }} +{{ end -}} + +{{- if .NoteGroups -}} +{{ range .NoteGroups -}} +{{ .Title }}: +{{ range .Notes }} +{{ .Body }} +{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} + +{{- if .Versions }} +[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD +{{ range .Versions -}} +{{ if .Tag.Previous -}} +[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }} +{{ end -}} +{{ end -}} +{{ end -}} diff --git a/.chglog/config.yml b/.chglog/config.yml new file mode 100644 index 0000000000..e9fff99ed6 --- /dev/null +++ b/.chglog/config.yml @@ -0,0 +1,48 @@ +style: github +template: CHANGELOG.tpl.md +info: + title: CHANGELOG + repository_url: https://github.com/terraform-aws-modules/terraform-aws-eks +options: + + commits: + sort_by: Type + filters: + Type: + - feat + - fix + - improvement + - docs + - refactor + - test + - ci + - chore + + commit_groups: + group_by: Type + title_maps: + feat: FEATURES + fix: BUG FIXES + improvement: ENHANCEMENTS + docs: DOCS + refactor: REFACTORS + test: TESTS + ci: CI + chore: CHORES + + header: + pattern: "^(\\.+)\\s*:\\s*(.+)$" + pattern_maps: + - Type + - Subject + + notes: + keywords: + - BREAKING CHANGE + - NOTES + + refs: + actions: + - Closes + - Fixes + - Resolves diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000000..0323f79ab1 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contributing + +When contributing to this repository, please first discuss the change you wish to make via issue, +email, or any other method with the owners of this repository before making a change. + +Please note we have a code of conduct, please follow it in all your interactions with the project. + +## Pull Request Process + +1. Ensure any install or build dependencies are removed before the end of the layer when doing a build. +2. Update the README.md with details of changes to the interface, this includes new environment variables, exposed ports, useful file locations and container parameters. +3. Once all outstanding comments and checklist items have been addressed, your contribution will be merged! Merged PRs will be included in the next release. The terraform-aws-eks mainteners takes care of updating the CHANGELOG as they merge. + +## Checklists for contributions + +- [ ] Add [sementics prefix](#semantic-pull-requests) to your PR or Commits (at leats one of your commit groups) +- [ ] CI tests are passing +- [ ] README.md has been updated after any changes to variables and outputs. See https://github.com/terraform-aws-modules/terraform-aws-eks/#doc-generation + +## Semantic Pull Requests + +To generate changelog, Pull Requests or Commits must have sementic and must follow conventional specs below: + +- `feat:` for new features +- `fix:` for bug fixes +- `improvement:` for enhancements +- `docs:` for documentation and examples +- `refactor:` for code refactoring +- `test:` for tests +- `ci:` for CI purpose +- `chore:` for chores stuff + +There is a special prefix `skip changelog` which is skipped during changelog generation. It can be used for `skip changelog: update changelog` commit message by example. + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9310a74454..6a10d8fa5f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,6 +6,5 @@ Please explain the changes you made here and link to any relevant issues. ### Checklist -- [ ] Change added to CHANGELOG.md. All changes must be added and breaking changes and highlighted - [ ] CI tests are passing - [ ] README.md has been updated after any changes to variables and outputs. See https://github.com/terraform-aws-modules/terraform-aws-eks/#doc-generation diff --git a/.github/semantic.yml b/.github/semantic.yml new file mode 100644 index 0000000000..e0c83c7166 --- /dev/null +++ b/.github/semantic.yml @@ -0,0 +1,14 @@ +# By default types specified in commitizen/conventional-commit-types is used. +# See: https://github.com/commitizen/conventional-commit-types/blob/v2.3.0/index.json +# You can override the valid types +types: + - feat + - fix + - improvement + - docs + - refactor + - test + - ci + - chore + - skip changelog + - skip ci diff --git a/.github/workflows/changelog-check.yaml b/.github/workflows/changelog-check.yaml new file mode 100644 index 0000000000..34fe6056c3 --- /dev/null +++ b/.github/workflows/changelog-check.yaml @@ -0,0 +1,20 @@ +name: CHANGELOG Checks +on: + pull_request: + paths: + - CHANGELOG.md + +jobs: + changelog-check: + name: CHANGELOG Check + runs-on: ubuntu-latest + steps: + - name: Fail the check if changelog change + run: |- + echo "Thank you for your contribution!" + echo "" + echo "The 'CHANGELOG.md' file contents are handled by the maintainers during merge. This is to prevent pull request merge conflicts." + echo "Please see the Contributing Guide for additional pull request review items." + echo "" + echo "Remove any changes to the 'CHANGELOG.md' file and commit them in this pull request." + exit 1 diff --git a/CHANGELOG.md b/CHANGELOG.pre-v11.0.0.md similarity index 85% rename from CHANGELOG.md rename to CHANGELOG.pre-v11.0.0.md index 343031df6c..4d49358a52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.pre-v11.0.0.md @@ -5,15 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## Next release - -## [[v10.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v10.0.0...HEAD)] - 2020-xx-xx - -- Write your awesome change here (GH-xxxx) - -## History - -## [[v10.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...v10.0.0)] - 2020-03-12 +## [v10.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...v10.0.0) - 2020-03-12 BREAKING CHANGES: @@ -39,7 +31,7 @@ NOTES: From EKS 1.15, the VPC tag `kubernetes.io/cluster/: shared` is no longer required. So we droped those tags from exemples. -## [[v9.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...v9.0.0)] - 2020-02-27 +## [v9.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...v9.0.0) - 2020-02-27 - **Breaking:** Removal of autoscaling IAM policy and tags (by @max-rocket-internet) - Revert #631. Add back manage security group flags. (by @ryanooi) @@ -57,19 +49,19 @@ From EKS 1.15, the VPC tag `kubernetes.io/cluster/: shared` is no Autoscaling policy and tags have been removed from this module. This reduces complexity and increases security as the policy was attached to the node group IAM role. To manage it outside of this module either follow the example in `examples/irsa` to attach an IAM role to the cluster-autoscaler `serviceAccount` or create [the policy](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/v8.2.0/workers.tf#L361-L416) outside this module and pass it in using the `workers_additional_policies` variable. -## [[v8.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...v8.2.0)] - 2020-01-29 +## [v8.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...v8.2.0) - 2020-01-29 - Include ability to configure custom os-specific command for waiting until kube cluster is healthy (@sanjeevgiri) - Disable creation of ingress rules if worker nodes security groups are exists (@andjelx) - [CI] Update pre-commit and re-generate docs to work with terraform-docs >= 0.8.1 (@barryib) -## [[v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0)] - 2020-01-17 +## [v8.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0) - 2020-01-17 - Fix index reference on destroy for output `oidc_provider_arn` (@stevie-) - Add support for restricting access to the public API endpoint (@sidprak) - Add an `ignore_lifecycle` rule to prevent Terraform from scaling down ASG behind AWS EKS Managed Node Group (by @davidalger) -## [[v8.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v7.0.1)] - 2020-01-09 +## [v8.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v7.0.1) - 2020-01-09 - **Breaking:** Change logic of security group whitelisting. Will always whitelist worker security group on control plane security group either provide one or create new one. See Important notes below for upgrade notes (by @ryanooi) - **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. See Important notes below for upgrade notes (by @sdehaes) @@ -110,14 +102,14 @@ For security group whitelisting change. After upgrade, have to remove `cluster_c terraform import module.eks.aws_security_group_rule.cluster_https_worker_ingress _ingress_tcp_443_443_ ``` -## [[v7.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.1...v7.0.0)] - 2019-12-11 +## [v7.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.1...v7.0.0) - 2019-12-11 - Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) - Updated `instance_profile_names` and `instance_profile_arns` outputs to also consider launch template as well as asg (by @ankitwal) - Fix broken terraform plan/apply on a cluster < 1.14 (by @hodduc) - Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) -## [[v7.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v7.0.0)] - 2019-10-30 +## [v7.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v7.0.0) - 2019-10-30 - **Breaking:** Allow for specifying a custom AMI for the worker nodes. (by @bmcstdio) - Added support for Windows workers AMIs (by @hodduc) @@ -131,20 +123,20 @@ terraform import module.eks.aws_security_group_rule.cluster_https_worker_ingress An AMI is now specified using the whole name, for example `amazon-eks-node-1.14-v20190927`. -## [[v6.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2)] - 2019-10-07 +## [v6.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2) - 2019-10-07 - Added `tags` to `aws_eks_cluster` introduced by terraform-provider-aws 2.31.0 (by @morganchristiansson) - Add option to enable lifecycle hooks creation (by @barryib) - Remove helm chart value `sslCertPath` described in `docs/autoscaling.md` (by @wi1dcard) - Attaching of IAM policies for autoscaler and CNI to the worker nodes now optional (by @dpiddockcmp) -## [[v6.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...v6.0.1)] - 2019-09-25 +## [v6.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...v6.0.1) - 2019-09-25 - Added support for different workers AMI's, i.e. with GPU support (by @rvoitenko) - Use null as default value for `target_group_arns` attribute of worker autoscaling group (by @tatusl) - Output empty string when cluster identity is empty (by @tbarry) -## [[v6.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v6.0.0)] - 2019-09-17 +## [v6.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v6.0.0) - 2019-09-17 - Added `market_type` to `workers_launch_template.tf` allow the usage of spot nodegroups without mixed instances policy. - Added support for log group tag in `./cluster.tf` (@lucas-giaco) @@ -172,11 +164,11 @@ You will need to move worker groups from `worker_groups_launch_template_mixed` t Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [""]`. -## [[v5.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v5.1.1)] - 2019-07-30 +## [v5.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v5.1.1) - 2019-07-30 - Added new tag in `worker.tf` with autoscaling_enabled = true flag (by @insider89) -## [[v5.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...v5.1.0)] - 2019-07-30 +## [v5.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...v5.1.0) - 2019-07-30 - Option to set a KMS key for the log group and encrypt it (by @till-krauss) - Output the name of the cloudwatch log group (by @gbooth27) @@ -186,7 +178,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Update default override instance types to work with Cluster Autoscaler (by @nauxliu on behalf of RightCapital) - Examples now specify `enable_dns_hostnames = true`, as per [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html) (by @karolinepauls) -## [[v5.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...v5.0.0)] - 2019-06-19 +## [v5.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...v5.0.0) - 2019-06-19 - Added Termination Policy Option to worker ASGs (by @undeadops) - Update EBS optimized instances type (by @gloutsch) @@ -200,17 +192,17 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Change override instance_types to list (by @nauxliu on behalf of RightCapital) - Fix toggle for IAM instance profile creation for mixed launch templates (by @jnozo) -## [[v4.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.1...v4.0.2)] - 2019-05-07 +## [v4.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.1...v4.0.2) - 2019-05-07 - Added 2 new examples, also tidy up basic example (by @max-rocket-internet) - Updates to travis, PR template (by @max-rocket-internet) - Fix typo in data.tf (by @max-rocket-internet) - Add missing launch template items in `aws_auth.tf` (by @max-rocket-internet) -## [[v4.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1)] - 2019-05-07 +## [v4.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1) - 2019-05-07 - Fix annoying typo: worker_group_xx vs worker_groups_xx (by @max-rocket-internet) -## [[v4.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v3.0.0...v4.0.0)] - 2019-05-07 +## [v4.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v3.0.0...v4.0.0) - 2019-05-07 - Added support for custom service linked role for Auto Scaling group (by @voanhduy1512) - Added support for custom IAM roles for cluster and workers (by @erks) @@ -225,7 +217,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Switch to https for the pre-commit repos (by @rothandrew) - Add instructions on how to enable the docker bridge network (by @rothandrew) -## [[v3.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.1...v3.0.0)] - 2019-04-15 +## [v3.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.1...v3.0.0) - 2019-04-15 - Fixed: Ability to destroy clusters due to security groups being attached to ENI's (by @whiskeyjimbo) - Added outputs for worker IAM instance profile(s) (by @soapergem) @@ -233,13 +225,13 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Updated vpc module version and aws provider version. (by @chenrui333) - Upgraded default kubernetes version from 1.11 to 1.12 (by @stijndehaes) -## [[v2.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.0...v2.3.1)] - 2019-03-26 +## [v2.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.0...v2.3.1) - 2019-03-26 - Added support for eks public and private endpoints (by @stijndehaes) - Added minimum inbound traffic rule to the cluster worker security group as per the [EKS security group requirements](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) (by @sc250024) - (Breaking Change) Replaced `enable_docker_bridge` with a generic option called `bootstrap_extra_args` to resolve [310](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/310) (by @max-rocket-internet) -## [[v2.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.3.0)] - 2019-03-20 +## [v2.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.3.0) - 2019-03-20 - Allow additional policies to be attached to worker nodes (by @rottenbytes) - Ability to specify a placement group for each worker group (by @matheuss) @@ -251,13 +243,13 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Added optional name filter variable to be able to pin worker AMI to a release (by @max-rocket-internet) - Added `--enable-docker-bridge` option for bootstrap.sh in AMI (by @michaelmccord) -## [[v2.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.2.2)] - 2019-02-25 +## [v2.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.2.2) - 2019-02-25 - Ability to specify a path for IAM roles (by @tekn0ir) -## [[v2.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.0...v2.2.1)] - 2019-02-18 +## [v2.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.0...v2.2.1) - 2019-02-18 -## [[v2.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.1.0...v2.2.0)] - 2019-02-07 +## [v2.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.1.0...v2.2.0) - 2019-02-07 - Ability to specify a permissions_boundary for IAM roles (by @dylanhellems) - Ability to configure force_delete for the worker group ASG (by @stefansedich) @@ -269,7 +261,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Fixed a bug where worker group defaults were being used for launch template user data (by @leonsodhi-lf) - Managed_aws_auth option is true, the aws-auth configmap file is no longer created, and write_aws_auth_config must be set to true to generate config_map. (by @yutachaos) -## [[v2.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.0.0...v2.1.0)] - 2019-01-15 +## [v2.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.0.0...v2.1.0) - 2019-01-15 - Initial support for worker groups based on Launch Templates (by @skang0601) - Updated the `update_config_map_aws_auth` resource to trigger when the EKS cluster endpoint changes. This likely means that a new cluster was spun up so our ConfigMap won't exist (fixes #234) (by @elatt) @@ -277,7 +269,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Fixed zsh-specific syntax in retry loop for aws auth config map (by @marcelloromani) - Fix: fail deployment if applying the aws auth config map still fails after 10 attempts (by @marcelloromani) -## [[v2.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.8.0...v2.0.0)] - 2018-12-14 +## [v2.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.8.0...v2.0.0) - 2018-12-14 - (Breaking Change) New input variables `map_accounts_count`, `map_roles_count` and `map_users_count` to allow using computed values as part of `map_accounts`, `map_roles` and `map_users` configs (by @chili-man on behalf of OpenGov). - (Breaking Change) New variables `cluster_create_security_group` and `worker_create_security_group` to stop `value of 'count' cannot be computed` error. @@ -288,7 +280,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Travis fixes (by @RothAndrew) - Fixed some Windows compatibility issues (by @RothAndrew) -## [[v1.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.7.0...v1.8.0)] - 2018-12-04 +## [v1.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.7.0...v1.8.0) - 2018-12-04 - Support for using AWS Launch Templates to define autoscaling groups (by @skang0601) - `suspended_processes` to `worker_groups` input (by @bkmeneguello) @@ -302,7 +294,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Removed providers from `main.tf`. (by @max-rocket-internet) - Removed `configure_kubectl_session` references in documentation [#171](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/171) (by @dominik-k) -## [[v1.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.6.0...v1.7.0)] - 2018-10-09 +## [v1.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.6.0...v1.7.0) - 2018-10-09 - Worker groups can be created with a specified IAM profile. (from @laverya) - exposed `aws_eks_cluster` create and destroy timeouts (by @RGPosadas) @@ -313,7 +305,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Cosmetic syntax changes to improve readability. (by @max-rocket-internet) - add `protect_from_scale_in` to solve issue #134 (by @kinghajj) -## [[v1.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.5.0...v1.6.0)] - 2018-09-04 +## [v1.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.5.0...v1.6.0) - 2018-09-04 - add support for [`amazon-eks-node-*` AMI with bootstrap script](https://aws.amazon.com/blogs/opensource/improvements-eks-worker-node-provisioning/) (by @erks) - expose `kubelet_extra_args` worker group option (replacing `kubelet_node_labels`) to allow specifying arbitrary kubelet options (e.g. taints and labels) (by @erks) @@ -324,7 +316,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - **Breaking change** Removed support for `eks-worker-*` AMI. The cluster specifying a custom AMI based off of `eks-worker-*` AMI will have to rebuild the AMI from `amazon-eks-node-*`. (by @erks) - **Breaking change** Removed `kubelet_node_labels` worker group option in favor of `kubelet_extra_args`. (by @erks) -## [[v1.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.4.0...v1.5.0)] - 2018-08-30 +## [v1.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.4.0...v1.5.0) - 2018-08-30 - add spot_price option to aws_launch_configuration - add enable_monitoring option to aws_launch_configuration @@ -335,20 +327,20 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ If you are upgrading from 1.4 you should fix state after upgrade: `terraform state rm module.eks.data.http.workstation_external_ip` - Can now selectively override keys in `workers_group_defaults` variable rather than callers maintaining a duplicate of the whole map. (by @dpiddockcmp) -## [[v1.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.3.0...v1.4.0)] - 2018-08-02 +## [v1.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.3.0...v1.4.0) - 2018-08-02 - manage eks workers' root volume size and type. - `workers_asg_names` added to outputs. (kudos to @laverya) - New top level variable `worker_group_count` added to replace the use of `length(var.worker_groups)`. This allows using computed values as part of worker group configs. (complaints to @laverya) -## [[v1.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.2.0...v1.3.0)] - 2018-07-11 +## [v1.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.2.0...v1.3.0) - 2018-07-11 - New variables `map_accounts`, `map_roles` and `map_users` in order to manage additional entries in the `aws-auth` configmap. (by @max-rocket-internet) - kubelet_node_labels worker group option allows setting --node-labels= in kubelet. (Hat-tip, @bshelton229 👒) - `worker_iam_role_arn` added to outputs. Sweet, @hatemosphere 🔥 - Worker subnets able to be specified as a dedicated list per autoscaling group. (up top, @bshelton229 🙏) -## [[v1.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.1.0...v1.2.0)] - 2018-07-01 +## [v1.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.1.0...v1.2.0) - 2018-07-01 - new variable `pre_userdata` added to worker launch configuration allows to run scripts before the plugin does anything. (W00t, @jimbeck 🦉) - kubeconfig made much more flexible. (Bang up job, @sdavids13 💥) @@ -356,7 +348,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - Providing security groups didn't behave as expected. This has been fixed. (Good catch, @jimbeck 🔧) - workstation cidr to be allowed by created security group is now more flexible. (A welcome addition, @jimbeck 🔐) -## [[v1.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.0.0...v1.1.0)] - 2018-06-25 +## [v1.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.0.0...v1.1.0) - 2018-06-25 - new variable `worker_sg_ingress_from_port` allows to change the minimum port number from which pods will accept communication (Thanks, @ilyasotkov 👏). - expanded on worker example to show how multiple worker autoscaling groups can be created. @@ -367,13 +359,13 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - `kube-proxy` restart no longer needed in userdata. Good catch, @hatemosphere 🔥 - worker ASG reattachment wasn't possible when using `name`. Moved to `name_prefix` to allow recreation of resources. Kudos again, @hatemosphere 🐧 -## [[v1.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.2.0...v1.0.0)] - 2018-06-11 +## [v1.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.2.0...v1.0.0) - 2018-06-11 - security group id can be provided for either/both of the cluster and the workers. If not provided, security groups will be created with sufficient rules to allow cluster-worker communication. - kudos to @tanmng on the idea ⭐ - outputs of security group ids and worker ASG arns added for working with these resources outside the module. - Worker build out refactored to allow multiple autoscaling groups each having differing specs. If none are given, a single ASG is created with a set of sane defaults - big thanks to @kppullin 🥨 -## [[v0.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.1...v0.2.0)] - 2018-06-08 +## [v0.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.1...v0.2.0) - 2018-06-08 - ability to specify extra userdata code to execute following kubelet services start. - EBS optimization used whenever possible for the given instance type. @@ -381,7 +373,7 @@ Map roles need to rename `role_arn` to `rolearn` and `group = ""` to `groups = [ - files rendered from dedicated templates to separate out raw code and config from `hcl` - `workers_ami_id` is now made optional. If not specified, the module will source the latest AWS supported EKS AMI instead. -## [[v0.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.0...v0.1.1)] - 2018-06-07 +## [v0.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.0...v0.1.1) - 2018-06-07 - Pre-commit hooks fixed and working. - Made progress on CI, advancing the build to the final `kitchen test` stage before failing. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index b81ce2db90..0000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,13 +0,0 @@ -# Contributing - -When contributing to this repository, please first discuss the change you wish to make via issue, -email, or any other method with the owners of this repository before making a change. - -Please note we have a code of conduct, please follow it in all your interactions with the project. - -## Pull Request Process - -1. Ensure any install or build dependencies are removed before the end of the layer when doing a build. -2. Update the README.md with details of changes to the interface, this includes new environment variables, exposed ports, useful file locations and container parameters. -3. Increase the version numbers in any examples files and the README.md to the new version that this Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). -4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you do not have permission to do that, you may request the second reviewer to merge it for you. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..92c13f71f7 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +.PHONY: changelog release + +SEMTAG=tools/semtag + +CHANGELOG_FILE=CHANGELOG.md +TAG_QUERY=v11.0.0.. + +scope ?= "minor" + +changelog-unrelease: + git-chglog -o $(CHANGELOG_FILE) $(TAG_QUERY) + +changelog: + git-chglog -o $(CHANGELOG_FILE) --next-tag `$(SEMTAG) final -s $(scope) -o -f` $(TAG_QUERY) + +release: + $(SEMTAG) final -s $(scope) diff --git a/README.md b/README.md index a21694b5ce..930961a988 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,8 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws ## Change log -The [changelog](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/CHANGELOG.md) captures all important release notes. +- The [changelog](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/CHANGELOG.md) captures all important release notes from v11.0.0 +- For older release notes, refer to [changelog.pre-v11.0.0.md](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/CHANGELOG.pre-v11.0.0.md) ## Authors diff --git a/tools/semtag b/tools/semtag new file mode 100755 index 0000000000..568d4241ad --- /dev/null +++ b/tools/semtag @@ -0,0 +1,627 @@ +#!/usr/bin/env bash +# +# Thanks to @pnikosis for this script https://github.com/pnikosis/semtag +# +PROG=semtag +PROG_VERSION="v0.1.0" + +SEMVER_REGEX="^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(\-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$" +IDENTIFIER_REGEX="^\-([0-9A-Za-z-]+)\.([0-9A-Za-z-]+)*$" + +# Global variables +FIRST_VERSION="v0.0.0" +finalversion=$FIRST_VERSION +lastversion=$FIRST_VERSION +hasversiontag="false" +scope="patch" +displayonly="false" +forcetag="false" +forcedversion= +versionname= +identifier= + +HELP="\ +Usage: + $PROG + $PROG getlast + $PROG getfinal + $PROG (final|alpha|beta|candidate) [-s (major|minor|patch|auto) | -o] + $PROG --help + $PROG --version +Options: + -s The scope that must be increased, can be major, minor or patch. + The resulting version will match X.Y.Z(-PRERELEASE)(+BUILD) + where X, Y and Z are positive integers, PRERELEASE is an optionnal + string composed of alphanumeric characters describing if the build is + a release candidate, alpha or beta version, with a number. + BUILD is also an optional string composed of alphanumeric + characters and hyphens. + Setting the scope as 'auto', the script will chose the scope between + 'minor' and 'patch', depending on the amount of lines added (<10% will + choose patch). + -v Specifies manually the version to be tagged, must be a valid semantic version + in the format X.Y.Z where X, Y and Z are positive integers. + -o Output the version only, shows the bumped version, but doesn't tag. + -f Forces to tag, even if there are unstaged or uncommited changes. +Commands: + --help Print this help message. + --version Prints the program's version. + get Returns both current final version and last tagged version. + getlast Returns the latest tagged version. + getfinal Returns the latest tagged final version. + getcurrent Returns the current version, based on the latest one, if there are uncommited or + unstaged changes, they will be reflected in the version, adding the number of + pending commits, current branch and commit hash. + final Tags the current build as a final version, this only can be done on the master branch. + candidate Tags the current build as a release candidate, the tag will contain all + the commits from the last final version. + alpha Tags the current build as an alpha version, the tag will contain all + the commits from the last final version. + beta Tags the current build as a beta version, the tag will contain all + the commits from the last final version." + +# Commands and options +ACTION="getlast" +ACTION="$1" +shift + +# We get the parameters +while getopts "v:s:of" opt; do + case $opt in + v) + forcedversion="$OPTARG" + ;; + s) + scope="$OPTARG" + ;; + o) + displayonly="true" + ;; + f) + forcetag="true" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +# Gets a string with the version and returns an array of maximum size of 5 with all the parts of the sematinc version +# $1 The string containing the version in semantic format +# $2 The variable to store the result array: +# position 0: major number +# position 1: minor number +# position 2: patch number +# position 3: identifier (or prerelease identifier) +# position 4: build info +function explode_version { + local __version=$1 + local __result=$2 + if [[ $__version =~ $SEMVER_REGEX ]] ; then + local __major=${BASH_REMATCH[1]} + local __minor=${BASH_REMATCH[2]} + local __patch=${BASH_REMATCH[3]} + local __prere=${BASH_REMATCH[4]} + local __build=${BASH_REMATCH[5]} + eval "$__result=(\"$__major\" \"$__minor\" \"$__patch\" \"$__prere\" \"$__build\")" + else + eval "$__result=" + fi +} + +# Compare two versions and returns -1, 0 or 1 +# $1 The first version to compare +# $2 The second version to compare +# $3 The variable where to store the result +function compare_versions { + local __first + local __second + explode_version $1 __first + explode_version $2 __second + local lv=$3 + + # Compares MAJOR, MINOR and PATCH + for i in 0 1 2; do + local __numberfirst=${__first[$i]} + local __numbersecond=${__second[$i]} + case $(($__numberfirst - $__numbersecond)) in + 0) + ;; + -[0-9]*) + eval "$lv=-1" + return 0 + ;; + [0-9]*) + eval "$lv=1" + return 0 + ;; + esac + done + + # Identifiers should compare with the ASCII order. + local __identifierfirst=${__first[3]} + local __identifiersecond=${__second[3]} + if [[ -n "$__identifierfirst" ]] && [[ -n "$__identifiersecond" ]]; then + if [[ "$__identifierfirst" > "$__identifiersecond" ]]; then + eval "$lv=1" + return 0 + elif [[ "$__identifierfirst" < "$__identifiersecond" ]]; then + eval "$lv=-1" + return 0 + fi + elif [[ -z "$__identifierfirst" ]] && [[ -n "$__identifiersecond" ]]; then + eval "$lv=1" + return 0 + elif [[ -n "$__identifierfirst" ]] && [[ -z "$__identifiersecond" ]]; then + eval "$lv=-1" + return 0 + fi + + eval "$lv=0" +} + +# Returns the last version of two +# $1 The first version to compare +# $2 The second version to compare +# $3 The variable where to store the last one +function get_latest_of_two { + local __first=$1 + local __second=$2 + local __result + local __latest=$3 + compare_versions $__first $__second __result + case $__result in + 0) + eval "$__latest=$__second" + ;; + -1) + eval "$__latest=$__second" + ;; + 1) + eval "$__latest=$__first" + ;; + esac +} + +# Assigns a 2 size array with the identifier, having the identifier at pos 0, and the number in pos 1 +# $1 The identifier in the format -id.# +# $2 The vferiable where to store the 2 size array +function explode_identifier { + local __identifier=$1 + local __result=$2 + if [[ $__identifier =~ $IDENTIFIER_REGEX ]] ; then + local __id=${BASH_REMATCH[1]} + local __number=${BASH_REMATCH[2]} + if [[ -z "$__number" ]]; then + __number=1 + fi + eval "$__result=(\"$__id\" \"$__number\")" + else + eval "$__result=" + fi +} + +# Gets a list of tags and assigns the base and latest versions +# Receives an array with the tags containing the versions +# Assigns to the global variables finalversion and lastversion the final version and the latest version +function get_latest { + local __taglist=("$@") + local __tagsnumber=${#__taglist[@]} + local __current + case $__tagsnumber in + 0) + finalversion=$FIRST_VERSION + lastversion=$FIRST_VERSION + ;; + 1) + __current=${__taglist[0]} + explode_version $__current ver + if [ -n "$ver" ]; then + if [ -n "${ver[3]}" ]; then + finalversion=$FIRST_VERSION + else + finalversion=$__current + fi + lastversion=$__current + else + finalversion=$FIRST_VERSION + lastversion=$FIRST_VERSION + fi + ;; + *) + local __lastpos=$(($__tagsnumber-1)) + for i in $(seq 0 $__lastpos) + do + __current=${__taglist[i]} + explode_version ${__taglist[i]} ver + if [ -n "$ver" ]; then + if [ -z "${ver[3]}" ]; then + get_latest_of_two $finalversion $__current finalversion + get_latest_of_two $lastversion $finalversion lastversion + else + get_latest_of_two $lastversion $__current lastversion + fi + fi + done + ;; + esac + + if git rev-parse -q --verify "refs/tags/$lastversion" >/dev/null; then + hasversiontag="true" + else + hasversiontag="false" + fi +} + +# Gets the next version given the provided scope +# $1 The version that is going to be bumped +# $2 The scope to bump +# $3 The variable where to stoer the result +function get_next_version { + local __exploded + local __fromversion=$1 + local __scope=$2 + local __result=$3 + explode_version $__fromversion __exploded + case $__scope in + major) + __exploded[0]=$((${__exploded[0]}+1)) + __exploded[1]=0 + __exploded[2]=0 + ;; + minor) + __exploded[1]=$((${__exploded[1]}+1)) + __exploded[2]=0 + ;; + patch) + __exploded[2]=$((${__exploded[2]}+1)) + ;; + esac + + eval "$__result=v${__exploded[0]}.${__exploded[1]}.${__exploded[2]}" +} + +function bump_version { + ## First we try to get the next version based on the existing last one + if [ "$scope" == "auto" ]; then + get_scope_auto scope + fi + + local __candidatefromlast=$FIRST_VERSION + local __explodedlast + explode_version $lastversion __explodedlast + if [[ -n "${__explodedlast[3]}" ]]; then + # Last version is not final + local __idlast + explode_identifier ${__explodedlast[3]} __idlast + + # We get the last, given the desired id based on the scope + __candidatefromlast="v${__explodedlast[0]}.${__explodedlast[1]}.${__explodedlast[2]}" + if [[ -n "$identifier" ]]; then + local __nextid="$identifier.1" + if [ "$identifier" == "${__idlast[0]}" ]; then + # We target the same identifier as the last so we increase one + __nextid="$identifier.$(( ${__idlast[1]}+1 ))" + __candidatefromlast="$__candidatefromlast-$__nextid" + else + # Different identifiers, we make sure we are assigning a higher identifier, if not, we increase the version + __candidatefromlast="$__candidatefromlast-$__nextid" + local __comparedwithlast + compare_versions $__candidatefromlast $lastversion __comparedwithlast + if [ "$__comparedwithlast" == -1 ]; then + get_next_version $__candidatefromlast $scope __candidatefromlast + __candidatefromlast="$__candidatefromlast-$__nextid" + fi + fi + fi + fi + + # Then we try to get the version based on the latest final one + local __candidatefromfinal=$FIRST_VERSION + get_next_version $finalversion $scope __candidatefromfinal + if [[ -n "$identifier" ]]; then + __candidatefromfinal="$__candidatefromfinal-$identifier.1" + fi + + # Finally we compare both candidates + local __resultversion + local __result + compare_versions $__candidatefromlast $__candidatefromfinal __result + case $__result in + 0) + __resultversion=$__candidatefromlast + ;; + -1) + __resultversion="$__candidatefromfinal" + ;; + 1) + __resultversion=$__candidatefromlast + ;; + esac + + eval "$1=$__resultversion" +} + +function increase_version { + local __version= + + if [ -z $forcedversion ]; then + bump_version __version + else + if [[ $forcedversion =~ $SEMVER_REGEX ]] ; then + compare_versions $forcedversion $lastversion __result + if [ $__result -le 0 ]; then + echo "Version can't be lower than last version: $lastversion" + exit 1 + fi + else + echo "Non valid version to bump" + exit 1 + fi + __version=$forcedversion + fi + + if [ "$displayonly" == "true" ]; then + echo "$__version" + else + if [ "$forcetag" == "false" ]; then + check_git_dirty_status + fi + local __commitlist + if [ "$finalversion" == "$FIRST_VERSION" ] || [ "$hasversiontag" != "true" ]; then + __commitlist="$(git log --pretty=oneline | cat)" + else + __commitlist="$(git log --pretty=oneline $finalversion... | cat)" + fi + + # If we are forcing a bump, we add bump to the commit list + if [[ -z $__commitlist && "$forcetag" == "true" ]]; then + __commitlist="bump" + fi + + if [[ -z $__commitlist ]]; then + echo "No commits since the last final version, not bumping version" + else + if [[ -z $versionname ]]; then + versionname=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + fi + local __message="$versionname +$__commitlist" + + # We check we have info on the user + local __username=$(git config user.name) + if [ -z "$__username" ]; then + __username=$(id -u -n) + git config user.name $__username + fi + local __useremail=$(git config user.email) + if [ -z "$__useremail" ]; then + __useremail=$(hostname) + git config user.email "$__username@$__useremail" + fi + + git tag -a $__version -m "$__message" + + # If we have a remote, we push there + local __remotes=$(git remote) + if [[ -n $__remotes ]]; then + for __remote in $__remotes; do + git push $__remote $__version > /dev/null + if [ $? -eq 0 ]; then + echo "$__version pushed to $__remote" + else + echo "Error pushing the tag $__version to $__remote" + exit 1 + fi + done + else + echo "$__version" + fi + fi + fi +} + +function check_git_dirty_status { + local __repostatus= + get_work_tree_status __repostatus + + if [ "$__repostatus" == "uncommitted" ]; then + echo "ERROR: You have uncommitted changes" + git status --porcelain + exit 1 + fi + + if [ "$__repostatus" == "unstaged" ]; then + echo "ERROR: You have unstaged changes" + git status --porcelain + exit 1 + fi +} + +# Get the total amount of lines of code in the repo +function get_total_lines { + local __empty_id="$(git hash-object -t tree /dev/null)" + local __changes="$(git diff --numstat $__empty_id | cat)" + local __added_deleted=$1 + get_changed_lines "$__changes" $__added_deleted +} + +# Get the total amount of lines of code since the provided tag +function get_sincetag_lines { + local __sincetag=$1 + local __changes="$(git diff --numstat $__sincetag | cat)" + local __added_deleted=$2 + get_changed_lines "$__changes" $__added_deleted +} + +function get_changed_lines { + local __changes_numstat=$1 + local __result=$2 + IFS=$'\n' read -rd '' -a __changes_array <<<"$__changes_numstat" + local __diff_regex="^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+.+$" + + local __total_added=0 + local __total_deleted=0 + for i in "${__changes_array[@]}" + do + if [[ $i =~ $__diff_regex ]] ; then + local __added=${BASH_REMATCH[1]} + local __deleted=${BASH_REMATCH[2]} + __total_added=$(( $__total_added+$__added )) + __total_deleted=$(( $__total_deleted+$__deleted )) + fi + done + eval "$2=( $__total_added $__total_deleted )" +} + +function get_scope_auto { + local __verbose=$2 + local __total=0 + local __since=0 + local __scope= + + get_total_lines __total + get_sincetag_lines $finalversion __since + + local __percentage=0 + if [ "$__total" != "0" ]; then + local __percentage=$(( 100*$__since/$__total )) + if [ $__percentage -gt "10" ]; then + __scope="minor" + else + __scope="patch" + fi + fi + + eval "$1=$__scope" + if [[ -n "$__verbose" ]]; then + echo "[Auto Scope] Percentage of lines changed: $__percentage" + echo "[Auto Scope] : $__scope" + fi +} + +function get_work_tree_status { + # Update the index + git update-index -q --ignore-submodules --refresh > /dev/null + eval "$1=" + + if ! git diff-files --quiet --ignore-submodules -- > /dev/null + then + eval "$1=unstaged" + fi + + if ! git diff-index --cached --quiet HEAD --ignore-submodules -- > /dev/null + then + eval "$1=uncommitted" + fi +} + +function get_current { + if [ "$hasversiontag" == "true" ]; then + local __commitcount="$(git rev-list $lastversion.. --count)" + else + local __commitcount="$(git rev-list --count HEAD)" + fi + local __status= + get_work_tree_status __status + + if [ "$__commitcount" == "0" ] && [ -z "$__status" ]; then + eval "$1=$lastversion" + else + local __buildinfo="$(git rev-parse --short HEAD)" + local __currentbranch="$(git rev-parse --abbrev-ref HEAD)" + if [ "$__currentbranch" != "master" ]; then + __buildinfo="$__currentbranch.$__buildinfo" + fi + + local __suffix= + if [ "$__commitcount" != "0" ]; then + if [ -n "$__suffix" ]; then + __suffix="$__suffix." + fi + __suffix="$__suffix$__commitcount" + fi + if [ -n "$__status" ]; then + if [ -n "$__suffix" ]; then + __suffix="$__suffix." + fi + __suffix="$__suffix$__status" + fi + + __suffix="$__suffix+$__buildinfo" + if [ "$lastversion" == "$finalversion" ]; then + scope="patch" + identifier= + local __bumped= + bump_version __bumped + eval "$1=$__bumped-dev.$__suffix" + else + eval "$1=$lastversion.$__suffix" + fi + fi +} + +function init { + git fetch > /dev/null + TAGS="$(git tag)" + IFS=$'\n' read -rd '' -a TAG_ARRAY <<<"$TAGS" + + get_latest ${TAG_ARRAY[@]} + currentbranch="$(git rev-parse --abbrev-ref HEAD)" +} + +case $ACTION in + --help) + echo -e "$HELP" + ;; + --version) + echo -e "${PROG}: $PROG_VERSION" + ;; + final) + init + diff=$(git diff master | cat) + if [ "$forcetag" == "false" ]; then + if [ -n "$diff" ]; then + echo "ERROR: Branch must be updated with master for final versions" + exit 1 + fi + fi + increase_version + ;; + alpha|beta) + init + identifier="$ACTION" + increase_version + ;; + candidate) + init + identifier="rc" + increase_version + ;; + getlast) + init + echo "$lastversion" + ;; + getfinal) + init + echo "$finalversion" + ;; + getcurrent) + init + get_current current + echo "$current" + ;; + get) + init + echo "Current final version: $finalversion" + echo "Last tagged version: $lastversion" + ;; + *) + echo "'$ACTION' is not a valid command, see --help for available commands." + ;; +esac From 3c064aa2a975a2fa71a7bd9e5944ff60a26c5b2e Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 17 Mar 2020 15:16:57 +0100 Subject: [PATCH 157/962] fix: fix git-chglog configuration (#802) --- .chglog/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chglog/config.yml b/.chglog/config.yml index e9fff99ed6..4e569d466c 100644 --- a/.chglog/config.yml +++ b/.chglog/config.yml @@ -31,7 +31,7 @@ options: chore: CHORES header: - pattern: "^(\\.+)\\s*:\\s*(.+)$" + pattern: "^(.+)\\s*:\\s*(.+)$" pattern_maps: - Type - Subject From a6583a33714723ea71da26eb5951eef6afbfd5fa Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 17 Mar 2020 15:29:59 +0100 Subject: [PATCH 158/962] fix: fix git-chglog template to format changelog `Type` nicely (#803) --- .chglog/CHANGELOG.tpl.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.chglog/CHANGELOG.tpl.md b/.chglog/CHANGELOG.tpl.md index 52d22241c0..7662c4d6c7 100644 --- a/.chglog/CHANGELOG.tpl.md +++ b/.chglog/CHANGELOG.tpl.md @@ -14,14 +14,14 @@ project adheres to [Semantic Versioning](http://semver.org/). {{ range .Commits -}} {{- if .Subject -}} - {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} -{{- end -}} +{{ end -}} {{ end }} {{ end -}} {{ else }} {{ range .Unreleased.Commits -}} {{- if .Subject -}} - {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} -{{- end -}} +{{ end -}} {{ end }} {{ end -}} {{ end -}} @@ -35,14 +35,14 @@ project adheres to [Semantic Versioning](http://semver.org/). {{ range .Commits -}} {{- if .Subject -}} - {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} -{{- end -}} +{{ end -}} {{ end }} {{ end -}} {{ else }} {{ range .Commits -}} {{- if .Subject -}} - {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} -{{- end -}} +{{ end -}} {{ end }} {{ end -}} From cd8a825a5a89743fa051263121507e15982eae93 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Tue, 17 Mar 2020 16:36:34 +0100 Subject: [PATCH 159/962] test: remove unused kitchen test related stuff (#787) Co-authored-by: Thierno IB. BARRY --- .gitignore | 2 -- .kitchen.yml | 20 -------------------- .ruby-version | 1 - Gemfile | 9 --------- README.md | 22 ---------------------- test/integration/default/test_eks.rb | 9 --------- 6 files changed, 63 deletions(-) delete mode 100644 .kitchen.yml delete mode 100644 .ruby-version delete mode 100644 Gemfile delete mode 100644 test/integration/default/test_eks.rb diff --git a/.gitignore b/.gitignore index 6128b4ba23..6b31084d05 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,6 @@ .terraform/ **/inspec.lock *.gem -.kitchen/ -.kitchen.local.yml Gemfile.lock terraform.tfstate.d/ eks-admin-cluster-role-binding.yaml diff --git a/.kitchen.yml b/.kitchen.yml deleted file mode 100644 index b185344d90..0000000000 --- a/.kitchen.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -driver: - name: "terraform" - root_module_directory: "examples/basic" - -provisioner: - name: "terraform" - -platforms: - - name: "aws" - -verifier: - name: "awspec" - -suites: - - name: "default" - verifier: - name: "awspec" - patterns: - - "test/integration/default/test_eks.rb" diff --git a/.ruby-version b/.ruby-version deleted file mode 100644 index 7bf4b6a8ae..0000000000 --- a/.ruby-version +++ /dev/null @@ -1 +0,0 @@ -2.4.6 diff --git a/Gemfile b/Gemfile deleted file mode 100644 index 57a0c15c31..0000000000 --- a/Gemfile +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -ruby '2.4.6' - -source 'https://rubygems.org/' do - gem 'awspec', '~> 1.4.2' - gem 'kitchen-terraform', '~> 3.2' - gem 'kitchen-verifier-awspec', '~> 0.1.1' -end diff --git a/README.md b/README.md index 930961a988..3bac8faaeb 100644 --- a/README.md +++ b/README.md @@ -94,28 +94,6 @@ module "eks" { * [IAM Permissions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md): Minimum IAM permissions needed to setup EKS Cluster. * [FAQ](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md): Frequently Asked Questions -## Testing - -This module has been packaged with [awspec](https://github.com/k1LoW/awspec) tests through [kitchen](https://kitchen.ci/) and [kitchen-terraform](https://newcontext-oss.github.io/kitchen-terraform/). To run them: - -1. Install [rvm](https://rvm.io/rvm/install) and the ruby version specified in the [Gemfile](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/Gemfile). -2. Install bundler and the gems from our Gemfile: - - ```bash - gem install bundler && bundle install - ``` - -3. Ensure your AWS environment is configured (i.e. credentials and region) for test. -4. Test using `bundle exec kitchen test` from the root of the repo. - -For now, connectivity to the kubernetes cluster is not tested but will be in the -future. Once the test fixture has converged, you can query the test cluster from -that terminal session with -```bash -kubectl get nodes --watch --kubeconfig kubeconfig -``` -(using default settings `config_output_path = "./"` & `write_kubeconfig = true`) - ## Doc generation Code formatting and documentation for variables and outputs is generated using [pre-commit-terraform hooks](https://github.com/antonbabenko/pre-commit-terraform) which uses [terraform-docs](https://github.com/segmentio/terraform-docs). diff --git a/test/integration/default/test_eks.rb b/test/integration/default/test_eks.rb deleted file mode 100644 index acf27844f6..0000000000 --- a/test/integration/default/test_eks.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -require 'awspec' - -# rubocop:disable LineLength -state_file = 'terraform.tfstate.d/kitchen-terraform-default-aws/terraform.tfstate' -tf_state = JSON.parse(File.open(state_file).read) -region = tf_state['modules'][0]['outputs']['region']['value'] -ENV['AWS_REGION'] = region From 2c98a00b21100238580283a05a7ca5222b91b712 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Tue, 17 Mar 2020 19:06:09 +0100 Subject: [PATCH 160/962] improvement: Add timeout to default wait_for_cluster_cmd (#791) --- README.md | 2 +- variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3bac8faaeb..70e6225d60 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | -| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"until wget --no-check-certificate -O - -q $ENDPOINT/healthz \u003e/dev/null; do sleep 4; done"` | no | +| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do wget --no-check-certificate -O - -q $ENDPOINT/healthz \u003e/dev/null \u0026\u0026 exit 0 \|\| true; sleep 5; done; echo TIMEOUT \u0026\u0026 exit 1"` | no | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | | worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | diff --git a/variables.tf b/variables.tf index 925b7b3109..c77bdcd479 100644 --- a/variables.tf +++ b/variables.tf @@ -201,7 +201,7 @@ variable "cluster_delete_timeout" { variable "wait_for_cluster_cmd" { description = "Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT" type = string - default = "until wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null; do sleep 4; done" + default = "for i in `seq 1 60`; do wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; sleep 5; done; echo TIMEOUT && exit 1" } variable "cluster_create_security_group" { From e8a1ce17d8a664babafb0963569af61a77c4cd76 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Wed, 18 Mar 2020 08:54:57 +0100 Subject: [PATCH 161/962] fix: Create `false` and avoid waiting forever for a non-existent cluster to respond (#789) --- cluster.tf | 2 +- examples/create_false/main.tf | 30 ++++++++++++++++++++++++++++++ examples/create_false/variables.tf | 3 +++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 examples/create_false/main.tf create mode 100644 examples/create_false/variables.tf diff --git a/cluster.tf b/cluster.tf index 2e0ca6862a..c558e52a66 100644 --- a/cluster.tf +++ b/cluster.tf @@ -46,7 +46,7 @@ resource "aws_eks_cluster" "this" { } resource "null_resource" "wait_for_cluster" { - count = var.manage_aws_auth ? 1 : 0 + count = var.create_eks && var.manage_aws_auth ? 1 : 0 depends_on = [ aws_eks_cluster.this[0] diff --git a/examples/create_false/main.tf b/examples/create_false/main.tf new file mode 100644 index 0000000000..86604aacdd --- /dev/null +++ b/examples/create_false/main.tf @@ -0,0 +1,30 @@ +provider "aws" { + region = var.region +} + +data "aws_eks_cluster" "cluster" { + count = 0 + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + count = 0 + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = element(concat(data.aws_eks_cluster.cluster[*].endpoint, list("")), 0) + cluster_ca_certificate = base64decode(element(concat(data.aws_eks_cluster.cluster[*].certificate_authority.0.data, list("")), 0)) + token = element(concat(data.aws_eks_cluster_auth.cluster[*].token, list("")), 0) + load_config_file = false + version = "~> 1.11" +} + +module "eks" { + source = "../.." + create_eks = false + + vpc_id = "" + cluster_name = "" + subnets = [] +} diff --git a/examples/create_false/variables.tf b/examples/create_false/variables.tf new file mode 100644 index 0000000000..81b8dbe73e --- /dev/null +++ b/examples/create_false/variables.tf @@ -0,0 +1,3 @@ +variable "region" { + default = "us-west-2" +} From 3957a7c02c9bf1d3f392bdf31a8f13af3bd7df25 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Wed, 18 Mar 2020 08:58:19 +0100 Subject: [PATCH 162/962] improvement: Generate `aws-auth` configmap's roles from Object. No more string concat. (#790) Do not use string concat to generate a YAML data structure Co-authored-by: Thierno IB. BARRY --- aws_auth.tf | 114 +++++++++++++++++++++----------------- templates/worker-role.tpl | 8 --- 2 files changed, 62 insertions(+), 60 deletions(-) delete mode 100644 templates/worker-role.tpl diff --git a/aws_auth.tf b/aws_auth.tf index 487763b68e..6c41f753e0 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -1,52 +1,61 @@ data "aws_caller_identity" "current" { } -data "template_file" "launch_template_worker_role_arns" { - count = var.create_eks ? local.worker_group_launch_template_count : 0 - template = file("${path.module}/templates/worker-role.tpl") +locals { + auth_launch_template_worker_roles = [ + for index in range(0, var.create_eks ? local.worker_group_launch_template_count : 0) : { + worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element( + coalescelist( + aws_iam_instance_profile.workers_launch_template.*.role, + data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, + ), + index + )}" + platform = lookup( + var.worker_groups_launch_template[index], + "platform", + local.workers_group_defaults["platform"] + ) + } + ] - vars = { - worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element( - coalescelist( - aws_iam_instance_profile.workers_launch_template.*.role, - data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, - ), - count.index, - )}" - platform = lookup( - var.worker_groups_launch_template[count.index], - "platform", - local.workers_group_defaults["platform"] - ) - } -} - -data "template_file" "worker_role_arns" { - count = var.create_eks ? local.worker_group_count : 0 - template = file("${path.module}/templates/worker-role.tpl") - - vars = { - worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element( - coalescelist( - aws_iam_instance_profile.workers.*.role, - data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, - [""] - ), - count.index, - )}" - platform = lookup( - var.worker_groups[count.index], - "platform", - local.workers_group_defaults["platform"] - ) - } -} - -data "template_file" "node_group_arns" { - count = var.create_eks ? length(module.node_groups.aws_auth_roles) : 0 - template = file("${path.module}/templates/worker-role.tpl") + auth_worker_roles = [ + for index in range(0, var.create_eks ? local.worker_group_count : 0) : { + worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element( + coalescelist( + aws_iam_instance_profile.workers.*.role, + data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, + [""] + ), + index, + )}" + platform = lookup( + var.worker_groups[index], + "platform", + local.workers_group_defaults["platform"] + ) + } + ] - vars = module.node_groups.aws_auth_roles[count.index] + # Convert to format needed by aws-auth ConfigMap + configmap_roles = [ + for role in concat( + local.auth_launch_template_worker_roles, + local.auth_worker_roles, + module.node_groups.aws_auth_roles, + ) : + { + rolearn = role["worker_role_arn"] + username = "system:node:{{EC2PrivateDNSName}}" + groups = concat( + [ + "system:bootstrappers", + "system:nodes", + ], + role["platform"] == "windows" ? ["eks:kube-proxy-windows"] : [] + ) + } + ] } resource "kubernetes_config_map" "aws_auth" { @@ -59,12 +68,13 @@ resource "kubernetes_config_map" "aws_auth" { } data = { - mapRoles = < Date: Wed, 18 Mar 2020 09:04:17 +0100 Subject: [PATCH 163/962] fix: Do not recreate cluster when no SG given (#798) --- cluster.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.tf b/cluster.tf index c558e52a66..c5be4b657a 100644 --- a/cluster.tf +++ b/cluster.tf @@ -15,7 +15,7 @@ resource "aws_eks_cluster" "this" { tags = var.tags vpc_config { - security_group_ids = [local.cluster_security_group_id] + security_group_ids = compact([local.cluster_security_group_id]) subnet_ids = var.subnets endpoint_private_access = var.cluster_endpoint_private_access endpoint_public_access = var.cluster_endpoint_public_access From 89401f1623c7d88ccd41f9a1b9bf03802f2c3aab Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Wed, 18 Mar 2020 09:09:32 +0100 Subject: [PATCH 164/962] fix: Add node_group direct dependency on eks_cluster (#796) Setting `manage_aws_auth = false` removes the ordering dependency between node_group and eks_cluster generating an error on first apply. This fixes that use case. Fixes #793 --- node_groups.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_groups.tf b/node_groups.tf index 2c689c3818..5c2b92eb4e 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -4,7 +4,7 @@ data "null_data_source" "node_groups" { count = var.create_eks ? 1 : 0 inputs = { - cluster_name = var.cluster_name + cluster_name = aws_eks_cluster.this[0].name # Ensure these resources are created before "unlocking" the data source. # `depends_on` causes a refresh on every run so is useless here. From 0c1ed0e6e9f5dbf34dfde0549f4409ab6af66383 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Wed, 18 Mar 2020 13:56:51 +0100 Subject: [PATCH 165/962] Improvement: Require kubernetes provider >=1.11.1 (#784) BREAKING CHANGE: The terraform-aws-eks module now require at least kubernetes `1.11.1`. This may cause terraform to fail to init if users have set version = "1.10" like we had in the examples. --- README.md | 2 +- examples/basic/main.tf | 2 +- examples/irsa/main.tf | 2 +- examples/launch_templates/main.tf | 2 +- examples/managed_node_groups/main.tf | 2 +- examples/secrets_encryption/main.tf | 2 +- examples/spot_instances/main.tf | 2 +- versions.tf | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 70e6225d60..a5ee7f8d14 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | aws | >= 2.52.0 | -| kubernetes | >= 1.6.2 | +| kubernetes | >= 1.11.1 | | local | >= 1.2 | | null | >= 2.1 | | random | >= 2.1 | diff --git a/examples/basic/main.tf b/examples/basic/main.tf index ee1d9bc251..4ce49b95b9 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -36,7 +36,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "1.10" + version = "~> 1.11" } data "aws_availability_zones" "available" { diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index 0fd934d44f..849db9c28f 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -32,7 +32,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "1.10" + version = "~> 1.11" } data "aws_availability_zones" "available" {} diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index a8d0ec62fe..d76a13c561 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -36,7 +36,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "1.10" + version = "~> 1.11" } data "aws_availability_zones" "available" { diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index adb5cfe70c..eed19c24ff 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -36,7 +36,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "1.10" + version = "~> 1.11" } data "aws_availability_zones" "available" { diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index 5c4b861be0..1a639e1d20 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -36,7 +36,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "1.10" + version = "~> 1.11" } data "aws_availability_zones" "available" { diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index cfac972f57..5d6b5eee5c 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -36,7 +36,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "1.10" + version = "~> 1.11" } data "aws_availability_zones" "available" { diff --git a/versions.tf b/versions.tf index f73abd331e..fd052bbe15 100644 --- a/versions.tf +++ b/versions.tf @@ -7,6 +7,6 @@ terraform { null = ">= 2.1" template = ">= 2.1" random = ">= 2.1" - kubernetes = ">= 1.6.2" + kubernetes = ">= 1.11.1" } } From d75e493e27399c0ba39a370a62eeb2adf24bf18c Mon Sep 17 00:00:00 2001 From: betajobot Date: Wed, 18 Mar 2020 21:34:12 +0100 Subject: [PATCH 166/962] [ci skip] Create "LICENSE". --- LICENSE | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/LICENSE b/LICENSE index ab6dbd68ad..51fca54c2a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,19 +1,11 @@ -Copyright (c) 2018 Brandon O'Connor - Run at Scale +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + http://www.apache.org/licenses/LICENSE-2.0 -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. From b135f920a1b14e71c3ff21b4713ba1d2a0f673d1 Mon Sep 17 00:00:00 2001 From: betajobot Date: Wed, 18 Mar 2020 21:34:12 +0100 Subject: [PATCH 167/962] [ci skip] Create ".editorconfig". --- .editorconfig | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..88cb251903 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,30 @@ +# EditorConfig is awesome: http://EditorConfig.org +# Uses editorconfig to maintain consistent coding styles + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +charset = utf-8 +end_of_line = lf +indent_size = 2 +indent_style = space +insert_final_newline = true +max_line_length = 80 +trim_trailing_whitespace = true + +[*.{tf,tfvars}] +indent_size = 2 +indent_style = space + +[*.md] +max_line_length = 0 +trim_trailing_whitespace = false + +[Makefile] +tab_width = 2 +indent_style = tab + +[COMMIT_EDITMSG] +max_line_length = 0 From 0c23191cd374ac4868f77cd1c8906b31ab2ee36f Mon Sep 17 00:00:00 2001 From: Betajobot <28572351+betajobot@users.noreply.github.com> Date: Wed, 18 Mar 2020 21:56:21 +0100 Subject: [PATCH 168/962] skip changelog - Removed some files (#811) --- .prettierignore | 1 - CODE_OF_CONDUCT.md | 46 ---------------------------------------------- 2 files changed, 47 deletions(-) delete mode 100644 .prettierignore delete mode 100644 CODE_OF_CONDUCT.md diff --git a/.prettierignore b/.prettierignore deleted file mode 100644 index 08ce6f3485..0000000000 --- a/.prettierignore +++ /dev/null @@ -1 +0,0 @@ -**/*.* diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 7730bca7f3..0000000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,46 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment include: - -- Using welcoming and inclusive language -- Being respectful of differing viewpoints and experiences -- Gracefully accepting constructive criticism -- Focusing on what is best for the community -- Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -- The use of sexualized language or imagery and unwelcome sexual attention or advances -- Trolling, insulting/derogatory comments, and personal or political attacks -- Public or private harassment -- Publishing others' private information, such as a physical or electronic address, without explicit permission -- Other conduct which could reasonably be considered inappropriate in a professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at brandon@atscale.run. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] - -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ From 9951c87a86b02e0f61a4d1560ad2e6e9596000ed Mon Sep 17 00:00:00 2001 From: slimm609 Date: Thu, 19 Mar 2020 11:22:22 -0400 Subject: [PATCH 169/962] fix: add ip address when manage_aws_auth is true and public_access is false (#745) --- README.md | 1 + cluster.tf | 12 ++++++++++++ variables.tf | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/README.md b/README.md index a5ee7f8d14..1db97d7ec4 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | | cluster\_encryption\_config | Configuration block with encryption configuration for the cluster. See examples/secrets\_encryption/main.tf for example format |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | +| cluster\_endpoint\_private\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS private API server endpoint, when public access is disabled | `list(string)` |
[
"0.0.0.0/0"
]
| no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | | cluster\_endpoint\_public\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | | cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. | `string` | `""` | no | diff --git a/cluster.tf b/cluster.tf index c5be4b657a..cd5e6cd5c7 100644 --- a/cluster.tf +++ b/cluster.tf @@ -45,6 +45,18 @@ resource "aws_eks_cluster" "this" { ] } +resource "aws_security_group_rule" "cluster_private_access" { + count = var.create_eks && var.manage_aws_auth && var.cluster_endpoint_private_access && var.cluster_endpoint_public_access == false ? 1 : 0 + type = "ingress" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = var.cluster_endpoint_private_access_cidrs + + security_group_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id +} + + resource "null_resource" "wait_for_cluster" { count = var.create_eks && var.manage_aws_auth ? 1 : 0 diff --git a/variables.tf b/variables.tf index c77bdcd479..2bd5c6eff3 100644 --- a/variables.tf +++ b/variables.tf @@ -234,6 +234,12 @@ variable "iam_path" { default = "/" } +variable "cluster_endpoint_private_access_cidrs" { + description = "List of CIDR blocks which can access the Amazon EKS private API server endpoint, when public access is disabled" + type = list(string) + default = ["0.0.0.0/0"] +} + variable "cluster_endpoint_private_access" { description = "Indicates whether or not the Amazon EKS private API server endpoint is enabled." type = bool From e768c6c1038b8545fa7f4746dc6f04422783fee5 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 19 Mar 2020 21:43:20 +0100 Subject: [PATCH 170/962] ci: Restrict sementic PR to validate PR title only (#804) --- .github/semantic.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/semantic.yml b/.github/semantic.yml index e0c83c7166..d8a21f9f41 100644 --- a/.github/semantic.yml +++ b/.github/semantic.yml @@ -1,3 +1,17 @@ +# Always validate the PR title, and ignore the commits +titleOnly: true + +# Always validate all commits, and ignore the PR title +commitsOnly: false + +# Always validate the PR title AND all the commits +titleAndCommits: false + +# Require at least one commit to be valid +# this is only relevant when using commitsOnly: true or titleAndCommits: true, +# which validate all commits by default +anyCommit: false + # By default types specified in commitizen/conventional-commit-types is used. # See: https://github.com/commitizen/conventional-commit-types/blob/v2.3.0/index.json # You can override the valid types @@ -12,3 +26,7 @@ types: - chore - skip changelog - skip ci + +# Allow use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns") +# this is only relevant when using commitsOnly: true (or titleAndCommits: true) +allowMergeCommits: false From de1419b597f6db444146535a633b0e7a945bcd3d Mon Sep 17 00:00:00 2001 From: Viacheslav Vasilyev Date: Fri, 20 Mar 2020 22:34:33 +0200 Subject: [PATCH 171/962] feat: Add interpreter option to `wait_for_cluster_cmd` (#795) Co-authored-by: Vasilyev, Viacheslav Co-authored-by: Thierno IB. BARRY --- README.md | 1 + cluster.tf | 3 ++- variables.tf | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1db97d7ec4..e978504e61 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | | wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do wget --no-check-certificate -O - -q $ENDPOINT/healthz \u003e/dev/null \u0026\u0026 exit 0 \|\| true; sleep 5; done; echo TIMEOUT \u0026\u0026 exit 1"` | no | +| wait\_for\_cluster\_interpreter | Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy. | `list(string)` |
[
"/bin/sh",
"-c"
]
| no | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | | worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | diff --git a/cluster.tf b/cluster.tf index cd5e6cd5c7..b3897c2246 100644 --- a/cluster.tf +++ b/cluster.tf @@ -65,7 +65,8 @@ resource "null_resource" "wait_for_cluster" { ] provisioner "local-exec" { - command = var.wait_for_cluster_cmd + command = var.wait_for_cluster_cmd + interpreter = var.wait_for_cluster_interpreter environment = { ENDPOINT = aws_eks_cluster.this[0].endpoint } diff --git a/variables.tf b/variables.tf index 2bd5c6eff3..3d6f8fabd2 100644 --- a/variables.tf +++ b/variables.tf @@ -204,6 +204,12 @@ variable "wait_for_cluster_cmd" { default = "for i in `seq 1 60`; do wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; sleep 5; done; echo TIMEOUT && exit 1" } +variable "wait_for_cluster_interpreter" { + description = "Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy." + type = list(string) + default = ["/bin/sh", "-c"] +} + variable "cluster_create_security_group" { description = "Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`." type = bool From 49b0667cd91e15f7c98dba67ad7e5aa5a6f54b5d Mon Sep 17 00:00:00 2001 From: "Sam \"Bo\" Beckwith III" Date: Tue, 24 Mar 2020 17:19:41 -0500 Subject: [PATCH 172/962] fix: Fix destroy failure when talking to EKS endpoint on private network (#815) --- cluster.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster.tf b/cluster.tf index b3897c2246..55a1747cfe 100644 --- a/cluster.tf +++ b/cluster.tf @@ -61,7 +61,8 @@ resource "null_resource" "wait_for_cluster" { count = var.create_eks && var.manage_aws_auth ? 1 : 0 depends_on = [ - aws_eks_cluster.this[0] + aws_eks_cluster.this[0], + aws_security_group_rule.cluster_private_access, ] provisioner "local-exec" { From de00694a6323389a42f20240de5b3bf4775059d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Hidalgo=20Garc=C3=ADa?= Date: Tue, 24 Mar 2020 17:28:59 -0500 Subject: [PATCH 173/962] feat: Add support for additional volumes in launch templates and launch configurations (#800) Co-authored-by: Jaime Hidalgo Co-authored-by: Thierno IB. BARRY --- local.tf | 1 + workers.tf | 30 ++++++++++++++++++++++++++++++ workers_launch_template.tf | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/local.tf b/local.tf index 704b5cdcb2..4ea1b72ffb 100644 --- a/local.tf +++ b/local.tf @@ -64,6 +64,7 @@ locals { service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. platform = "linux" # Platform of workers. either "linux" or "windows" + additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device diff --git a/workers.tf b/workers.tf index ce2b27c601..2eed088c4f 100644 --- a/workers.tf +++ b/workers.tf @@ -223,6 +223,36 @@ resource "aws_launch_configuration" "workers" { delete_on_termination = true } + dynamic "ebs_block_device" { + for_each = lookup(var.worker_groups[count.index], "additional_ebs_volumes", local.workers_group_defaults["additional_ebs_volumes"]) + + content { + device_name = ebs_block_device.value.block_device_name + volume_size = lookup( + ebs_block_device.value, + "volume_size", + local.workers_group_defaults["root_volume_size"], + ) + volume_type = lookup( + ebs_block_device.value, + "volume_type", + local.workers_group_defaults["root_volume_type"], + ) + iops = lookup( + ebs_block_device.value, + "iops", + local.workers_group_defaults["root_iops"], + ) + encrypted = lookup( + ebs_block_device.value, + "encrypted", + local.workers_group_defaults["root_encrypted"], + ) + delete_on_termination = lookup(ebs_block_device.value, "delete_on_termination", true) + } + + } + lifecycle { create_before_destroy = true } diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 52ec738588..4eea118167 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -351,6 +351,43 @@ resource "aws_launch_template" "workers_launch_template" { } } + dynamic "block_device_mappings" { + for_each = lookup(var.worker_groups_launch_template[count.index], "additional_ebs_volumes", local.workers_group_defaults["additional_ebs_volumes"]) + content { + device_name = block_device_mappings.value.block_device_name + + ebs { + volume_size = lookup( + block_device_mappings.value, + "volume_size", + local.workers_group_defaults["root_volume_size"], + ) + volume_type = lookup( + block_device_mappings.value, + "volume_type", + local.workers_group_defaults["root_volume_type"], + ) + iops = lookup( + block_device_mappings.value, + "iops", + local.workers_group_defaults["root_iops"], + ) + encrypted = lookup( + block_device_mappings.value, + "encrypted", + local.workers_group_defaults["root_encrypted"], + ) + kms_key_id = lookup( + block_device_mappings.value, + "kms_key_id", + local.workers_group_defaults["root_kms_key_id"], + ) + delete_on_termination = lookup(block_device_mappings.value, "delete_on_termination", true) + } + } + + } + tag_specifications { resource_type = "volume" From 4cc57532846290d66b700a4c68d3ffd5364ee884 Mon Sep 17 00:00:00 2001 From: Albert Lloveras Date: Tue, 31 Mar 2020 01:09:41 +1100 Subject: [PATCH 174/962] improvement: Use `aws_partition` to build IAM policy ARNs (#820) * Use aws_partition to build IAM policy ARNs * Update broken link to CONTRIBUTING.md in README.md --- README.md | 2 +- data.tf | 2 +- local.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e978504e61..8363392e9b 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ And install `terraform-docs` with `go get github.com/segmentio/terraform-docs` o Report issues/questions/feature requests on in the [issues](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/new) section. -Full contributing [guidelines are covered here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/CONTRIBUTING.md). +Full contributing [guidelines are covered here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/.github/CONTRIBUTING.md). ## Change log diff --git a/data.tf b/data.tf index d945c52833..7e1aa918b4 100644 --- a/data.tf +++ b/data.tf @@ -222,4 +222,4 @@ data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instanc ) } -data "aws_region" "current" {} +data "aws_partition" "current" {} diff --git a/local.tf b/local.tf index 4ea1b72ffb..658290b06e 100644 --- a/local.tf +++ b/local.tf @@ -22,7 +22,7 @@ locals { default_ami_id_linux = coalesce(local.workers_group_defaults.ami_id, data.aws_ami.eks_worker.id) default_ami_id_windows = coalesce(local.workers_group_defaults.ami_id_windows, data.aws_ami.eks_worker_windows.id) - policy_arn_prefix = contains(["cn-northwest-1", "cn-north-1"], data.aws_region.current.name) ? "arn:aws-cn:iam::aws:policy" : "arn:aws:iam::aws:policy" + policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" workers_group_defaults_defaults = { name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. From c8195c53481e858b854e745258e9db14ae52f54d Mon Sep 17 00:00:00 2001 From: Ilya Nemakov Date: Mon, 30 Mar 2020 19:27:56 +0300 Subject: [PATCH 175/962] feat: Add instance tag specifications to Launch Template (#822) --- workers_launch_template.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 4eea118167..947bd19a6d 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -403,6 +403,21 @@ resource "aws_launch_template" "workers_launch_template" { ) } + tag_specifications { + resource_type = "instance" + + tags = merge( + { + "Name" = "${aws_eks_cluster.this[0].name}-${lookup( + var.worker_groups_launch_template[count.index], + "name", + count.index, + )}-eks_asg" + }, + var.tags, + ) + } + tags = var.tags lifecycle { From eaa4d2c697e78a3e65d1f877ba46a39c1daa5f2f Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 31 Mar 2020 00:53:35 +0200 Subject: [PATCH 176/962] skip changelog: Prepare for release 11.0.0 (#819) --- .chglog/config.yml | 2 +- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/.chglog/config.yml b/.chglog/config.yml index 4e569d466c..16a95bc100 100644 --- a/.chglog/config.yml +++ b/.chglog/config.yml @@ -4,7 +4,6 @@ info: title: CHANGELOG repository_url: https://github.com/terraform-aws-modules/terraform-aws-eks options: - commits: sort_by: Type filters: @@ -20,6 +19,7 @@ options: commit_groups: group_by: Type + sort_by: Type title_maps: feat: FEATURES fix: BUG FIXES diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..b5511b9e54 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,43 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) and this +project adheres to [Semantic Versioning](http://semver.org/). + + +## [Unreleased] + + + + +## [v11.0.0] - 2020-03-31 +FEATURES: +- Add instance tag specifications to Launch Template ([#822](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/822)) +- Add support for additional volumes in launch templates and launch configurations ([#800](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/800)) +- Add interpreter option to `wait_for_cluster_cmd` ([#795](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/795)) + +ENHANCEMENTS: +- Use `aws_partition` to build IAM policy ARNs ([#820](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/820)) +- Generate `aws-auth` configmap's roles from Object. No more string concat. ([#790](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/790)) +- Add timeout to default wait_for_cluster_cmd ([#791](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/791)) +- automate changelog management ([#786](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/786)) + +BUG FIXES: +- Fix destroy failure when talking to EKS endpoint on private network ([#815](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/815)) +- add ip address when manage_aws_auth is true and public_access is false ([#745](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/745)) +- Add node_group direct dependency on eks_cluster ([#796](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/796)) +- Do not recreate cluster when no SG given ([#798](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/798)) +- Create `false` and avoid waiting forever for a non-existent cluster to respond ([#789](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/789)) +- fix git-chglog template to format changelog `Type` nicely ([#803](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/803)) +- fix git-chglog configuration ([#802](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/802)) + +CI: +- Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) + +TESTS: +- remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) + + +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.0.0...HEAD +[v11.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v10.0.0...v11.0.0 From 2fd078e7c1c9df29c6c7a90989060bf84075af9d Mon Sep 17 00:00:00 2001 From: Valeri GOLUBEV Date: Sat, 11 Apr 2020 14:11:22 +0200 Subject: [PATCH 177/962] feat: Add support for EC2 principal in assume worker role policy for China (#827) * Add support for EC2 principal in assume worker role policy for China AWS * Remove local partition according to requested change Co-authored-by: Valeri GOLUBEV --- README.md | 18 +++++++++++++++--- aws_auth.tf | 4 ++-- data.tf | 3 ++- modules/node_groups/README.md | 6 +++++- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8363392e9b..71f5ba76e2 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,18 @@ Many thanks to [the contributors listed here](https://github.com/terraform-aws-m MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. +## Requirements + +| Name | Version | +|------|---------| +| terraform | >= 0.12.9 | +| aws | >= 2.52.0 | +| kubernetes | >= 1.11.1 | +| local | >= 1.2 | +| null | >= 2.1 | +| random | >= 2.1 | +| template | >= 2.1 | + ## Providers | Name | Version | @@ -138,7 +150,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a ## Inputs | Name | Description | Type | Default | Required | -|------|-------------|------|---------|:-----:| +|------|-------------|------|---------|:--------:| | attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | | cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | | cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | @@ -173,11 +185,11 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | | node\_groups | Map of map of node groups to create. See `node_groups` module's documentation for more details | `any` | `{}` | no | | node\_groups\_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentaton for more details | `any` | `{}` | no | -| permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | n/a | yes | +| permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | -| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do wget --no-check-certificate -O - -q $ENDPOINT/healthz \u003e/dev/null \u0026\u0026 exit 0 \|\| true; sleep 5; done; echo TIMEOUT \u0026\u0026 exit 1"` | no | +| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do wget --no-check-certificate -O - -q $ENDPOINT/healthz \u003e/dev/null \u0026\u0026 exit 0 || true; sleep 5; done; echo TIMEOUT \u0026\u0026 exit 1"` | no | | wait\_for\_cluster\_interpreter | Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy. | `list(string)` |
[
"/bin/sh",
"-c"
]
| no | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | diff --git a/aws_auth.tf b/aws_auth.tf index 6c41f753e0..d5e1c096bf 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -4,7 +4,7 @@ data "aws_caller_identity" "current" { locals { auth_launch_template_worker_roles = [ for index in range(0, var.create_eks ? local.worker_group_launch_template_count : 0) : { - worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element( + worker_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${element( coalescelist( aws_iam_instance_profile.workers_launch_template.*.role, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, @@ -21,7 +21,7 @@ locals { auth_worker_roles = [ for index in range(0, var.create_eks ? local.worker_group_count : 0) : { - worker_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${element( + worker_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${element( coalescelist( aws_iam_instance_profile.workers.*.role, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, diff --git a/data.tf b/data.tf index 7e1aa918b4..bb5effd111 100644 --- a/data.tf +++ b/data.tf @@ -5,6 +5,7 @@ locals { worker_ami_name_filter_windows = (var.worker_ami_name_filter_windows != "" ? var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${tonumber(var.cluster_version) >= 1.14 ? var.cluster_version : 1.14}-*" ) + ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" } data "aws_iam_policy_document" "workers_assume_role_policy" { @@ -17,7 +18,7 @@ data "aws_iam_policy_document" "workers_assume_role_policy" { principals { type = "Service" - identifiers = ["ec2.amazonaws.com"] + identifiers = [local.ec2_principal] } } } diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 6ac8842f35..dce434906f 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -34,6 +34,10 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | version | Kubernetes version | string | Provider default behavior | +## Requirements + +No requirements. + ## Providers | Name | Version | @@ -44,7 +48,7 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In ## Inputs | Name | Description | Type | Default | Required | -|------|-------------|------|---------|:-----:| +|------|-------------|------|---------|:--------:| | cluster\_name | Name of parent cluster | `string` | n/a | yes | | create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | default\_iam\_role\_arn | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | n/a | yes | From e41c4aa78a470af67d8677a67e673156a5f2715a Mon Sep 17 00:00:00 2001 From: sr-n <56052347+sr-n@users.noreply.github.com> Date: Wed, 22 Apr 2020 10:36:03 -0700 Subject: [PATCH 178/962] fix: Prevent `coalescelist` null argument error when destroying worker_group_launch_templates (#842) Resolves #841 --- aws_auth.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_auth.tf b/aws_auth.tf index d5e1c096bf..e2aac0e0fc 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -8,6 +8,7 @@ locals { coalescelist( aws_iam_instance_profile.workers_launch_template.*.role, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, + [""] ), index )}" From 058a516b0ff0b99eea06d9067e712f9818a1044c Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 22 Apr 2020 23:34:24 +0200 Subject: [PATCH 179/962] chore: Use chore semantic commits instead of skip changelog (#844) --- .chglog/config.yml | 2 -- .github/CONTRIBUTING.md | 2 +- .github/semantic.yml | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.chglog/config.yml b/.chglog/config.yml index 16a95bc100..af0993ac08 100644 --- a/.chglog/config.yml +++ b/.chglog/config.yml @@ -15,7 +15,6 @@ options: - refactor - test - ci - - chore commit_groups: group_by: Type @@ -28,7 +27,6 @@ options: refactor: REFACTORS test: TESTS ci: CI - chore: CHORES header: pattern: "^(.+)\\s*:\\s*(.+)$" diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 0323f79ab1..5ab6a91687 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -30,5 +30,5 @@ To generate changelog, Pull Requests or Commits must have sementic and must foll - `ci:` for CI purpose - `chore:` for chores stuff -There is a special prefix `skip changelog` which is skipped during changelog generation. It can be used for `skip changelog: update changelog` commit message by example. +The `chore` prefix skipped during changelog generation. It can be used for `chore: update changelog` commit message by example. diff --git a/.github/semantic.yml b/.github/semantic.yml index d8a21f9f41..376c06a6a8 100644 --- a/.github/semantic.yml +++ b/.github/semantic.yml @@ -24,8 +24,6 @@ types: - test - ci - chore - - skip changelog - - skip ci # Allow use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns") # this is only relevant when using commitsOnly: true (or titleAndCommits: true) From fed9c5bf19b14e9468ceb807ba9b3a16602ef8e8 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 22 Apr 2020 23:39:58 +0200 Subject: [PATCH 180/962] chore: update changelog (#845) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5511b9e54..c486905794 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,11 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +BUG FIXES: +- Prevent `coalescelist` null argument error when destroying worker_group_launch_templates ([#842](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/842)) +FEATURES: +- Add support for EC2 principal in assume worker role policy for China ([#827](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/827)) From 36d1b5afa5847193bd5205c09622da9f70b76eba Mon Sep 17 00:00:00 2001 From: keznikl Date: Thu, 23 Apr 2020 16:57:58 +0200 Subject: [PATCH 181/962] fix: Wrap `local.configmap_roles.groups` with tolist() to avoid panic (#846) --- aws_auth.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_auth.tf b/aws_auth.tf index e2aac0e0fc..db6dea4708 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -48,13 +48,13 @@ locals { { rolearn = role["worker_role_arn"] username = "system:node:{{EC2PrivateDNSName}}" - groups = concat( + groups = tolist(concat( [ "system:bootstrappers", "system:nodes", ], role["platform"] == "windows" ? ["eks:kube-proxy-windows"] : [] - ) + )) } ] } From 9c2a71874404fa371d740086eeb21da6755a2007 Mon Sep 17 00:00:00 2001 From: slimm609 Date: Thu, 23 Apr 2020 11:36:51 -0400 Subject: [PATCH 182/962] fix: Add `vpc_config.cluster_security_group` output as primary cluster security group id (#828) --- README.md | 3 ++- outputs.tf | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 71f5ba76e2..3d0743d2e7 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_iam\_role\_name | IAM role name of the EKS cluster. | | cluster\_id | The name/id of the EKS cluster. | | cluster\_oidc\_issuer\_url | The URL on the EKS cluster OIDC Issuer | -| cluster\_security\_group\_id | Security group ID attached to the EKS cluster. | +| cluster\_primary\_security\_group\_id | The cluster primary security group ID created by the EKS cluster on 1.14 or later. Referred to as 'Cluster security group' in the EKS console. | +| cluster\_security\_group\_id | Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console. | | cluster\_version | The Kubernetes server version for the EKS cluster. | | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | | kubeconfig | kubectl config file contents for this EKS cluster. | diff --git a/outputs.tf b/outputs.tf index b1f8c3c049..9a6e4033bf 100644 --- a/outputs.tf +++ b/outputs.tf @@ -24,7 +24,7 @@ output "cluster_version" { } output "cluster_security_group_id" { - description = "Security group ID attached to the EKS cluster." + description = "Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console." value = local.cluster_security_group_id } @@ -48,6 +48,11 @@ output "cluster_oidc_issuer_url" { value = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0] } +output "cluster_primary_security_group_id" { + description = "The cluster primary security group ID created by the EKS cluster on 1.14 or later. Referred to as 'Cluster security group' in the EKS console." + value = var.cluster_version >= 1.14 ? element(concat(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, list("")), 0) : null +} + output "cloudwatch_log_group_name" { description = "Name of cloudwatch log group created" value = aws_cloudwatch_log_group.this[*].name From 7afecf637905dc6aeb58519abaf4d09231e8c9a6 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 23 Apr 2020 22:14:15 +0200 Subject: [PATCH 183/962] chore: update changelog (#847) --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c486905794..303b4adfc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,14 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] + + + + +## [v11.1.0] - 2020-04-23 BUG FIXES: +- Add `vpc_config.cluster_security_group` output as primary cluster security group id ([#828](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/828)) +- Wrap `local.configmap_roles.groups` with tolist() to avoid panic ([#846](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/846)) - Prevent `coalescelist` null argument error when destroying worker_group_launch_templates ([#842](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/842)) FEATURES: @@ -43,5 +50,6 @@ TESTS: - remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.0.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.1.0...HEAD +[v11.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.0.0...v11.1.0 [v11.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v10.0.0...v11.0.0 From 25ebaab1cc88685902a971667f590d097e37c491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giuseppe=20Crin=C3=B2?= Date: Tue, 5 May 2020 13:22:43 +0200 Subject: [PATCH 184/962] Fix typo (#859) --- docs/iam-permissions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index f0e161140c..65eb3666c0 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -86,20 +86,20 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy", "iam:CreateInstanceProfile", - "iam:CreateOpenIDConnectProvider", + "iam:CreateOpenIDConnectProvider", "iam:CreateServiceLinkedRole", "iam:CreatePolicy", "iam:CreatePolicyVersion", "iam:CreateRole", "iam:DeleteInstanceProfile", - "iam:DeleteOpenIDConnectProvider", + "iam:DeleteOpenIDConnectProvider", "iam:DeletePolicy", "iam:DeleteRole", "iam:DeleteRolePolicy", "iam:DeleteServiceLinkedRole", "iam:DetachRolePolicy", "iam:GetInstanceProfile", - "iam:GetOpenIDConnectProvider", + "iam:GetOpenIDConnectProvider", "iam:GetPolicy", "iam:GetPolicyVersion", "iam:GetRole", From b183b972539fa9d39dc55c453d68a234fdb683b1 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Wed, 6 May 2020 10:54:14 +0200 Subject: [PATCH 185/962] improvement: Remove dependency on external template provider (#854) * Remove template_file for generating kubeconfig Push logic from terraform down to the template. Makes the formatting slightly easier to follow * Remove template_file for generating userdata Updates to the eks_cluster now do not trigger recreation of launch configurations * Remove template_file for LT userdata * Remove template dependency --- README.md | 2 - data.tf | 214 ++++++++++++++++--------------------- kubectl.tf | 2 +- outputs.tf | 6 +- templates/kubeconfig.tpl | 16 ++- versions.tf | 1 - workers.tf | 2 +- workers_launch_template.tf | 2 +- 8 files changed, 109 insertions(+), 136 deletions(-) diff --git a/README.md b/README.md index 3d0743d2e7..c4b751a3f7 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | local | >= 1.2 | | null | >= 2.1 | | random | >= 2.1 | -| template | >= 2.1 | ## Providers @@ -145,7 +144,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | local | >= 1.2 | | null | >= 2.1 | | random | >= 2.1 | -| template | >= 2.1 | ## Inputs diff --git a/data.tf b/data.tf index bb5effd111..e0dcc5e34e 100644 --- a/data.tf +++ b/data.tf @@ -66,138 +66,104 @@ data "aws_iam_policy_document" "cluster_assume_role_policy" { } } -data "template_file" "kubeconfig" { - count = var.create_eks ? 1 : 0 - template = file("${path.module}/templates/kubeconfig.tpl") - - vars = { - kubeconfig_name = local.kubeconfig_name - endpoint = aws_eks_cluster.this[0].endpoint - cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data - aws_authenticator_command = var.kubeconfig_aws_authenticator_command - aws_authenticator_command_args = length(var.kubeconfig_aws_authenticator_command_args) > 0 ? " - ${join( - "\n - ", - var.kubeconfig_aws_authenticator_command_args, - )}" : " - ${join( - "\n - ", - formatlist("\"%s\"", ["token", "-i", aws_eks_cluster.this[0].name]), - )}" - aws_authenticator_additional_args = length(var.kubeconfig_aws_authenticator_additional_args) > 0 ? " - ${join( - "\n - ", - var.kubeconfig_aws_authenticator_additional_args, - )}" : "" - aws_authenticator_env_variables = length(var.kubeconfig_aws_authenticator_env_variables) > 0 ? " env:\n${join( - "\n", - data.template_file.aws_authenticator_env_variables.*.rendered, - )}" : "" - } -} - -data "template_file" "aws_authenticator_env_variables" { - count = length(var.kubeconfig_aws_authenticator_env_variables) - - template = < 0 ? var.kubeconfig_aws_authenticator_command_args : ["token", "-i", aws_eks_cluster.this[0].name] + aws_authenticator_additional_args = var.kubeconfig_aws_authenticator_additional_args + aws_authenticator_env_variables = var.kubeconfig_aws_authenticator_env_variables + }) : "" + + userdata = [for worker in var.worker_groups : templatefile( + lookup( + worker, + "userdata_template_file", + lookup(worker, "platform", local.workers_group_defaults["platform"]) == "windows" ? "${path.module}/templates/userdata_windows.tpl" : "${path.module}/templates/userdata.sh.tpl" + ), + merge( + { + platform = lookup(worker, "platform", local.workers_group_defaults["platform"]) + cluster_name = aws_eks_cluster.this[0].name + endpoint = aws_eks_cluster.this[0].endpoint + cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data + pre_userdata = lookup( + worker, + "pre_userdata", + local.workers_group_defaults["pre_userdata"], + ) + additional_userdata = lookup( + worker, + "additional_userdata", + local.workers_group_defaults["additional_userdata"], + ) + bootstrap_extra_args = lookup( + worker, + "bootstrap_extra_args", + local.workers_group_defaults["bootstrap_extra_args"], + ) + kubelet_extra_args = lookup( + worker, + "kubelet_extra_args", + local.workers_group_defaults["kubelet_extra_args"], + ) + }, + lookup( + worker, + "userdata_template_extra_args", + local.workers_group_defaults["userdata_template_extra_args"] + ) ) - ) + ) if var.create_eks + ] - vars = merge({ - platform = lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) - cluster_name = aws_eks_cluster.this[0].name - endpoint = aws_eks_cluster.this[0].endpoint - cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data - pre_userdata = lookup( - var.worker_groups[count.index], - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - var.worker_groups[count.index], - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - var.worker_groups[count.index], - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - var.worker_groups[count.index], - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, + launch_template_userdata = [for worker in var.worker_groups_launch_template : templatefile( lookup( - var.worker_groups[count.index], - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) -} - -data "template_file" "launch_template_userdata" { - count = var.create_eks ? local.worker_group_launch_template_count : 0 - template = lookup( - var.worker_groups_launch_template[count.index], - "userdata_template_file", - file( - lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" + worker, + "userdata_template_file", + lookup(worker, "platform", local.workers_group_defaults["platform"]) == "windows" ? "${path.module}/templates/userdata_windows.tpl" : "${path.module}/templates/userdata.sh.tpl" + ), + merge( + { + platform = lookup(worker, "platform", local.workers_group_defaults["platform"]) + cluster_name = aws_eks_cluster.this[0].name + endpoint = aws_eks_cluster.this[0].endpoint + cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data + pre_userdata = lookup( + worker, + "pre_userdata", + local.workers_group_defaults["pre_userdata"], + ) + additional_userdata = lookup( + worker, + "additional_userdata", + local.workers_group_defaults["additional_userdata"], + ) + bootstrap_extra_args = lookup( + worker, + "bootstrap_extra_args", + local.workers_group_defaults["bootstrap_extra_args"], + ) + kubelet_extra_args = lookup( + worker, + "kubelet_extra_args", + local.workers_group_defaults["kubelet_extra_args"], + ) + }, + lookup( + worker, + "userdata_template_extra_args", + local.workers_group_defaults["userdata_template_extra_args"] + ) ) - ) - - vars = merge({ - platform = lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) - cluster_name = aws_eks_cluster.this[0].name - endpoint = aws_eks_cluster.this[0].endpoint - cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data - pre_userdata = lookup( - var.worker_groups_launch_template[count.index], - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - var.worker_groups_launch_template[count.index], - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - var.worker_groups_launch_template[count.index], - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - var.worker_groups_launch_template[count.index], - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - var.worker_groups_launch_template[count.index], - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) + ) if var.create_eks + ] } data "aws_iam_role" "custom_cluster_iam_role" { diff --git a/kubectl.tf b/kubectl.tf index 56aba1ffd9..1e660708d2 100644 --- a/kubectl.tf +++ b/kubectl.tf @@ -1,5 +1,5 @@ resource "local_file" "kubeconfig" { count = var.write_kubeconfig && var.create_eks ? 1 : 0 - content = data.template_file.kubeconfig[0].rendered + content = local.kubeconfig filename = substr(var.config_output_path, -1, 1) == "/" ? "${var.config_output_path}kubeconfig_${var.cluster_name}" : var.config_output_path } diff --git a/outputs.tf b/outputs.tf index 9a6e4033bf..8ec702f359 100644 --- a/outputs.tf +++ b/outputs.tf @@ -60,7 +60,7 @@ output "cloudwatch_log_group_name" { output "kubeconfig" { description = "kubectl config file contents for this EKS cluster." - value = concat(data.template_file.kubeconfig[*].rendered, [""])[0] + value = local.kubeconfig } output "kubeconfig_filename" { @@ -92,8 +92,8 @@ output "workers_asg_names" { output "workers_user_data" { description = "User data of worker groups" value = concat( - data.template_file.userdata.*.rendered, - data.template_file.launch_template_userdata.*.rendered, + local.userdata, + local.launch_template_userdata, ) } diff --git a/templates/kubeconfig.tpl b/templates/kubeconfig.tpl index 1696391e89..ad1beec739 100644 --- a/templates/kubeconfig.tpl +++ b/templates/kubeconfig.tpl @@ -23,6 +23,16 @@ users: apiVersion: client.authentication.k8s.io/v1alpha1 command: ${aws_authenticator_command} args: -${aws_authenticator_command_args} -${aws_authenticator_additional_args} -${aws_authenticator_env_variables} +%{~ for i in aws_authenticator_command_args } + - "${i}" +%{~ endfor ~} +%{ for i in aws_authenticator_additional_args } + - ${i} +%{~ endfor ~} +%{ if length(aws_authenticator_env_variables) > 0 } + env: + %{~ for k, v in aws_authenticator_env_variables ~} + - name: ${k} + value: ${v} + %{~ endfor ~} +%{ endif ~} diff --git a/versions.tf b/versions.tf index fd052bbe15..e5c8ce6043 100644 --- a/versions.tf +++ b/versions.tf @@ -5,7 +5,6 @@ terraform { aws = ">= 2.52.0" local = ">= 1.2" null = ">= 2.1" - template = ">= 2.1" random = ">= 2.1" kubernetes = ">= 1.11.1" } diff --git a/workers.tf b/workers.tf index 2eed088c4f..a1908b7391 100644 --- a/workers.tf +++ b/workers.tf @@ -170,7 +170,7 @@ resource "aws_launch_configuration" "workers" { "key_name", local.workers_group_defaults["key_name"], ) - user_data_base64 = base64encode(data.template_file.userdata.*.rendered[count.index]) + user_data_base64 = base64encode(local.userdata[count.index]) ebs_optimized = lookup( var.worker_groups[count.index], "ebs_optimized", diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 947bd19a6d..ebc997e9da 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -262,7 +262,7 @@ resource "aws_launch_template" "workers_launch_template" { local.workers_group_defaults["key_name"], ) user_data = base64encode( - data.template_file.launch_template_userdata.*.rendered[count.index], + local.launch_template_userdata[count.index], ) ebs_optimized = lookup( From 4a8c79053af1aefa64f2a01ca9d305220a03a69d Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 6 May 2020 12:16:39 +0200 Subject: [PATCH 186/962] docs: Remove useless template provider in examples (#863) --- examples/basic/main.tf | 4 ---- examples/irsa/main.tf | 4 ---- examples/launch_templates/main.tf | 4 ---- examples/managed_node_groups/main.tf | 4 ---- examples/secrets_encryption/main.tf | 4 ---- examples/spot_instances/main.tf | 4 ---- 6 files changed, 24 deletions(-) diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 4ce49b95b9..a21e530d2c 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -19,10 +19,6 @@ provider "null" { version = "~> 2.1" } -provider "template" { - version = "~> 2.1" -} - data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index 849db9c28f..bbb9e8a5a3 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -15,10 +15,6 @@ provider "null" { version = "~> 2.1" } -provider "template" { - version = "~> 2.1" -} - data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index d76a13c561..8d9cff8246 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -19,10 +19,6 @@ provider "null" { version = "~> 2.1" } -provider "template" { - version = "~> 2.1" -} - data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index eed19c24ff..1e7f802fa7 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -19,10 +19,6 @@ provider "null" { version = "~> 2.1" } -provider "template" { - version = "~> 2.1" -} - data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index 1a639e1d20..e4483539ca 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -19,10 +19,6 @@ provider "null" { version = "~> 2.1" } -provider "template" { - version = "~> 2.1" -} - data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 5d6b5eee5c..6cb45e70fd 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -19,10 +19,6 @@ provider "null" { version = "~> 2.1" } -provider "template" { - version = "~> 2.1" -} - data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } From 9bfdba9fb86290b562c096cb404df4cd4e13223e Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 6 May 2020 14:00:20 +0200 Subject: [PATCH 187/962] chore: Move all locals in locals.tf (#865) --- data.tf | 111 ----------------------------------------------------- local.tf | 114 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 115 deletions(-) diff --git a/data.tf b/data.tf index e0dcc5e34e..a574e83bc6 100644 --- a/data.tf +++ b/data.tf @@ -1,13 +1,3 @@ -locals { - worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" - - # Windows nodes are available from k8s 1.14. If cluster version is less than 1.14, fix ami filter to some constant to not fail on 'terraform plan'. - worker_ami_name_filter_windows = (var.worker_ami_name_filter_windows != "" ? - var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${tonumber(var.cluster_version) >= 1.14 ? var.cluster_version : 1.14}-*" - ) - ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" -} - data "aws_iam_policy_document" "workers_assume_role_policy" { statement { sid = "EKSWorkerAssumeRole" @@ -50,7 +40,6 @@ data "aws_ami" "eks_worker_windows" { owners = [var.worker_ami_owner_id_windows] } - data "aws_iam_policy_document" "cluster_assume_role_policy" { statement { sid = "EKSClusterAssumeRole" @@ -66,106 +55,6 @@ data "aws_iam_policy_document" "cluster_assume_role_policy" { } } -locals { - kubeconfig = var.create_eks ? templatefile("${path.module}/templates/kubeconfig.tpl", { - kubeconfig_name = local.kubeconfig_name - endpoint = aws_eks_cluster.this[0].endpoint - cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data - aws_authenticator_command = var.kubeconfig_aws_authenticator_command - aws_authenticator_command_args = length(var.kubeconfig_aws_authenticator_command_args) > 0 ? var.kubeconfig_aws_authenticator_command_args : ["token", "-i", aws_eks_cluster.this[0].name] - aws_authenticator_additional_args = var.kubeconfig_aws_authenticator_additional_args - aws_authenticator_env_variables = var.kubeconfig_aws_authenticator_env_variables - }) : "" - - userdata = [for worker in var.worker_groups : templatefile( - lookup( - worker, - "userdata_template_file", - lookup(worker, "platform", local.workers_group_defaults["platform"]) == "windows" - ? "${path.module}/templates/userdata_windows.tpl" - : "${path.module}/templates/userdata.sh.tpl" - ), - merge( - { - platform = lookup(worker, "platform", local.workers_group_defaults["platform"]) - cluster_name = aws_eks_cluster.this[0].name - endpoint = aws_eks_cluster.this[0].endpoint - cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data - pre_userdata = lookup( - worker, - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - worker, - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - worker, - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - worker, - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - worker, - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) - ) if var.create_eks - ] - - launch_template_userdata = [for worker in var.worker_groups_launch_template : templatefile( - lookup( - worker, - "userdata_template_file", - lookup(worker, "platform", local.workers_group_defaults["platform"]) == "windows" - ? "${path.module}/templates/userdata_windows.tpl" - : "${path.module}/templates/userdata.sh.tpl" - ), - merge( - { - platform = lookup(worker, "platform", local.workers_group_defaults["platform"]) - cluster_name = aws_eks_cluster.this[0].name - endpoint = aws_eks_cluster.this[0].endpoint - cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data - pre_userdata = lookup( - worker, - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - worker, - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - worker, - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - worker, - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - worker, - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) - ) if var.create_eks - ] -} - data "aws_iam_role" "custom_cluster_iam_role" { count = var.manage_cluster_iam_resources ? 0 : 1 name = var.cluster_iam_role_name diff --git a/local.tf b/local.tf index 658290b06e..e268a38f33 100644 --- a/local.tf +++ b/local.tf @@ -13,14 +13,22 @@ locals { cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id - default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] - kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name + default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] + default_ami_id_linux = coalesce(local.workers_group_defaults.ami_id, data.aws_ami.eks_worker.id) + default_ami_id_windows = coalesce(local.workers_group_defaults.ami_id_windows, data.aws_ami.eks_worker_windows.id) + + kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name worker_group_count = length(var.worker_groups) worker_group_launch_template_count = length(var.worker_groups_launch_template) - default_ami_id_linux = coalesce(local.workers_group_defaults.ami_id, data.aws_ami.eks_worker.id) - default_ami_id_windows = coalesce(local.workers_group_defaults.ami_id_windows, data.aws_ami.eks_worker_windows.id) + worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" + # Windows nodes are available from k8s 1.14. If cluster version is less than 1.14, fix ami filter to some constant to not fail on 'terraform plan'. + worker_ami_name_filter_windows = (var.worker_ami_name_filter_windows != "" ? + var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${tonumber(var.cluster_version) >= 1.14 ? var.cluster_version : 1.14}-*" + ) + + ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" workers_group_defaults_defaults = { @@ -128,4 +136,102 @@ locals { "t2.small", "t2.xlarge" ] + + kubeconfig = var.create_eks ? templatefile("${path.module}/templates/kubeconfig.tpl", { + kubeconfig_name = local.kubeconfig_name + endpoint = aws_eks_cluster.this[0].endpoint + cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data + aws_authenticator_command = var.kubeconfig_aws_authenticator_command + aws_authenticator_command_args = length(var.kubeconfig_aws_authenticator_command_args) > 0 ? var.kubeconfig_aws_authenticator_command_args : ["token", "-i", aws_eks_cluster.this[0].name] + aws_authenticator_additional_args = var.kubeconfig_aws_authenticator_additional_args + aws_authenticator_env_variables = var.kubeconfig_aws_authenticator_env_variables + }) : "" + + userdata = [for worker in var.worker_groups : templatefile( + lookup( + worker, + "userdata_template_file", + lookup(worker, "platform", local.workers_group_defaults["platform"]) == "windows" + ? "${path.module}/templates/userdata_windows.tpl" + : "${path.module}/templates/userdata.sh.tpl" + ), + merge( + { + platform = lookup(worker, "platform", local.workers_group_defaults["platform"]) + cluster_name = aws_eks_cluster.this[0].name + endpoint = aws_eks_cluster.this[0].endpoint + cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data + pre_userdata = lookup( + worker, + "pre_userdata", + local.workers_group_defaults["pre_userdata"], + ) + additional_userdata = lookup( + worker, + "additional_userdata", + local.workers_group_defaults["additional_userdata"], + ) + bootstrap_extra_args = lookup( + worker, + "bootstrap_extra_args", + local.workers_group_defaults["bootstrap_extra_args"], + ) + kubelet_extra_args = lookup( + worker, + "kubelet_extra_args", + local.workers_group_defaults["kubelet_extra_args"], + ) + }, + lookup( + worker, + "userdata_template_extra_args", + local.workers_group_defaults["userdata_template_extra_args"] + ) + ) + ) if var.create_eks + ] + + launch_template_userdata = [for worker in var.worker_groups_launch_template : templatefile( + lookup( + worker, + "userdata_template_file", + lookup(worker, "platform", local.workers_group_defaults["platform"]) == "windows" + ? "${path.module}/templates/userdata_windows.tpl" + : "${path.module}/templates/userdata.sh.tpl" + ), + merge( + { + platform = lookup(worker, "platform", local.workers_group_defaults["platform"]) + cluster_name = aws_eks_cluster.this[0].name + endpoint = aws_eks_cluster.this[0].endpoint + cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data + pre_userdata = lookup( + worker, + "pre_userdata", + local.workers_group_defaults["pre_userdata"], + ) + additional_userdata = lookup( + worker, + "additional_userdata", + local.workers_group_defaults["additional_userdata"], + ) + bootstrap_extra_args = lookup( + worker, + "bootstrap_extra_args", + local.workers_group_defaults["bootstrap_extra_args"], + ) + kubelet_extra_args = lookup( + worker, + "kubelet_extra_args", + local.workers_group_defaults["kubelet_extra_args"], + ) + }, + lookup( + worker, + "userdata_template_extra_args", + local.workers_group_defaults["userdata_template_extra_args"] + ) + ) + ) if var.create_eks + ] } From 99f59bfcc9c3b994c0adcced86bbea7796815fdd Mon Sep 17 00:00:00 2001 From: Scott Crooks Date: Wed, 6 May 2020 17:20:28 +0200 Subject: [PATCH 188/962] feat: Change EKS default version to 1.16 (#857) --- .pre-commit-config.yaml | 2 +- README.md | 4 ++-- docs/spot-instances.md | 8 ++++---- examples/spot_instances/main.tf | 2 +- variables.tf | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 71ccc545ba..e6874c6a0c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.27.0 + rev: v1.30.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index c4b751a3f7..1141c5d1dc 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ provider "kubernetes" { module "my-cluster" { source = "terraform-aws-modules/eks/aws" cluster_name = "my-cluster" - cluster_version = "1.14" + cluster_version = "1.16" subnets = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] vpc_id = "vpc-1234556abcdef" @@ -164,7 +164,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | -| cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | `"1.15"` | no | +| cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | `"1.16"` | no | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | | create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | eks\_oidc\_root\_ca\_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | diff --git a/docs/spot-instances.md b/docs/spot-instances.md index e8a31e45ab..1eb4da4257 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -32,7 +32,7 @@ Example worker group configuration that uses an ASG with launch configuration fo name = "on-demand-1" instance_type = "m4.xlarge" asg_max_size = 1 - kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=normal" + kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=normal" suspended_processes = ["AZRebalance"] }, { @@ -40,7 +40,7 @@ Example worker group configuration that uses an ASG with launch configuration fo spot_price = "0.199" instance_type = "c4.xlarge" asg_max_size = 20 - kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" + kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" suspended_processes = ["AZRebalance"] }, { @@ -48,7 +48,7 @@ Example worker group configuration that uses an ASG with launch configuration fo spot_price = "0.20" instance_type = "m4.xlarge" asg_max_size = 20 - kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" + kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" suspended_processes = ["AZRebalance"] } ] @@ -77,7 +77,7 @@ Launch Template support is a recent addition to both AWS and this module. It mig spot_instance_pools = 4 asg_max_size = 5 asg_desired_capacity = 5 - kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" + kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" public_ip = true }, ] diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 6cb45e70fd..007cd60357 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -71,7 +71,7 @@ module "eks" { spot_instance_pools = 4 asg_max_size = 5 asg_desired_capacity = 5 - kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot" + kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" public_ip = true }, ] diff --git a/variables.tf b/variables.tf index 3d6f8fabd2..b949524f05 100644 --- a/variables.tf +++ b/variables.tf @@ -28,7 +28,7 @@ variable "cluster_security_group_id" { variable "cluster_version" { description = "Kubernetes version to use for the EKS cluster." type = string - default = "1.15" + default = "1.16" } variable "config_output_path" { From 312596f46d937c74143ce9a9563438b2b52f0f0e Mon Sep 17 00:00:00 2001 From: graywolf-at-work <43144838+graywolf-at-work@users.noreply.github.com> Date: Wed, 6 May 2020 19:36:58 +0000 Subject: [PATCH 189/962] feat: Create kubeconfig with non-executable permissions (#864) * Create kubeconfig with non-executable permissions Kubeconfig does not really need to be executable, so let's not create it with executable bit set. * Bump tf version --- README.md | 4 ++-- kubectl.tf | 8 +++++--- versions.tf | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1141c5d1dc..ec4d1c124b 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | terraform | >= 0.12.9 | | aws | >= 2.52.0 | | kubernetes | >= 1.11.1 | -| local | >= 1.2 | +| local | >= 1.4 | | null | >= 2.1 | | random | >= 2.1 | @@ -141,7 +141,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a |------|---------| | aws | >= 2.52.0 | | kubernetes | >= 1.11.1 | -| local | >= 1.2 | +| local | >= 1.4 | | null | >= 2.1 | | random | >= 2.1 | diff --git a/kubectl.tf b/kubectl.tf index 1e660708d2..a83f0f95e7 100644 --- a/kubectl.tf +++ b/kubectl.tf @@ -1,5 +1,7 @@ resource "local_file" "kubeconfig" { - count = var.write_kubeconfig && var.create_eks ? 1 : 0 - content = local.kubeconfig - filename = substr(var.config_output_path, -1, 1) == "/" ? "${var.config_output_path}kubeconfig_${var.cluster_name}" : var.config_output_path + count = var.write_kubeconfig && var.create_eks ? 1 : 0 + content = local.kubeconfig + filename = substr(var.config_output_path, -1, 1) == "/" ? "${var.config_output_path}kubeconfig_${var.cluster_name}" : var.config_output_path + file_permission = "0644" + directory_permission = "0755" } diff --git a/versions.tf b/versions.tf index e5c8ce6043..21e74d2e97 100644 --- a/versions.tf +++ b/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = ">= 2.52.0" - local = ">= 1.2" + local = ">= 1.4" null = ">= 2.1" random = ">= 2.1" kubernetes = ">= 1.11.1" From 527d4bd3f512d39ace05728d1f34064a90824e15 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 7 May 2020 09:17:05 +0200 Subject: [PATCH 190/962] fix: Use splat syntax for cluster name to avoid `(known after apply)` in managed node groups (#868) --- node_groups.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_groups.tf b/node_groups.tf index 5c2b92eb4e..8ead35b89e 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -4,7 +4,7 @@ data "null_data_source" "node_groups" { count = var.create_eks ? 1 : 0 inputs = { - cluster_name = aws_eks_cluster.this[0].name + cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] # Ensure these resources are created before "unlocking" the data source. # `depends_on` causes a refresh on every run so is useless here. From a4e21e73cd3d1248ea3238722f4108c7686bedd8 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 8 May 2020 18:41:21 +0200 Subject: [PATCH 191/962] docs: Add notes for Kubernetes 1.16 (#873) BREAKING CHANGES: The default `cluster_version` is now 1.16. Kubernetes 1.16 includes a number of deprecated API removals, and you need to ensure your applications and add ons are updated, or workloads could fail after the upgrade is complete. For more information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). For action you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). Please set explicitly your `cluster_version` to an older EKS version until your workloads are ready for Kubernetes 1.16. --- .chglog/CHANGELOG.tpl.md | 13 +++++++++++-- .chglog/config.yml | 2 +- README.md | 6 ++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.chglog/CHANGELOG.tpl.md b/.chglog/CHANGELOG.tpl.md index 7662c4d6c7..01fa64d17a 100644 --- a/.chglog/CHANGELOG.tpl.md +++ b/.chglog/CHANGELOG.tpl.md @@ -24,6 +24,15 @@ project adheres to [Semantic Versioning](http://semver.org/). {{ end -}} {{ end }} {{ end -}} + +{{- if .Unreleased.NoteGroups -}} +{{ range .Unreleased.NoteGroups -}} +{{ .Title }}: +{{ range .Notes -}} +- {{ .Body }} +{{ end }} +{{ end -}} +{{ end -}} {{ end -}} {{ range .Versions }} @@ -49,8 +58,8 @@ project adheres to [Semantic Versioning](http://semver.org/). {{- if .NoteGroups -}} {{ range .NoteGroups -}} {{ .Title }}: -{{ range .Notes }} -{{ .Body }} +{{ range .Notes -}} +- {{ .Body }} {{ end }} {{ end -}} {{ end -}} diff --git a/.chglog/config.yml b/.chglog/config.yml index af0993ac08..99485033c3 100644 --- a/.chglog/config.yml +++ b/.chglog/config.yml @@ -36,7 +36,7 @@ options: notes: keywords: - - BREAKING CHANGE + - BREAKING CHANGES - NOTES refs: diff --git a/README.md b/README.md index ec4d1c124b..f6f62991fc 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,12 @@ Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aw * You want these resources to exist within security groups that allow communication and coordination. These can be user provided or created within the module. * You've created a Virtual Private Cloud (VPC) and subnets where you intend to put the EKS resources. The VPC satisfies [EKS requirements](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html). +## Important note + +The default `cluster_version`is now 1.16. Kubernetes 1.16 includes a number of deprecated API removals, and you need to ensure your applications and add ons are updated, or workloads could fail after the upgrade is complete. For more information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). For action you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html#1-16-prequisites). + +Please set explicitly your `cluster_version` to an older EKS version until your workloads are ready for Kubernetes 1.16. + ## Usage example A full example leveraging other community modules is contained in the [examples/basic directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). From bb822a1e56c00e3edb16bb64c6fa1e783d5ededb Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Sat, 9 May 2020 19:58:38 +0200 Subject: [PATCH 192/962] fix: Fix Launch Templates error with aws 2.61.0 (#875) --- local.tf | 4 ++-- workers_launch_template.tf | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/local.tf b/local.tf index e268a38f33..c72af075f6 100644 --- a/local.tf +++ b/local.tf @@ -68,7 +68,7 @@ locals { suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] target_group_arns = null # A list of Application LoadBalancer (ALB) target group ARNs to be associated to the autoscaling group enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] - placement_group = "" # The name of the placement group into which to launch the instances, if any. + placement_group = null # The name of the placement group into which to launch the instances, if any. service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. platform = "linux" # Platform of workers. either "linux" or "windows" @@ -78,7 +78,7 @@ locals { root_kms_key_id = "" # The KMS key to use when encrypting the root storage device launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group launch_template_placement_tenancy = "default" # The placement tenancy for instances - launch_template_placement_group = "" # The name of the placement group into which to launch the instances, if any. + launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. root_encrypted = false # Whether the volume should be encrypted or not eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs diff --git a/workers_launch_template.tf b/workers_launch_template.tf index ebc997e9da..bdf1efdfb0 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -294,17 +294,17 @@ resource "aws_launch_template" "workers_launch_template" { ) } - placement { - tenancy = lookup( - var.worker_groups_launch_template[count.index], - "launch_template_placement_tenancy", - local.workers_group_defaults["launch_template_placement_tenancy"], - ) - group_name = lookup( - var.worker_groups_launch_template[count.index], - "launch_template_placement_group", - local.workers_group_defaults["launch_template_placement_group"], - ) + dynamic placement { + for_each = lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"]) != null ? [lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])] : [] + + content { + tenancy = lookup( + var.worker_groups_launch_template[count.index], + "launch_template_placement_tenancy", + local.workers_group_defaults["launch_template_placement_tenancy"], + ) + group_name = placement.value + } } dynamic instance_market_options { From d4f392405643fde8a9364bd1bb2bac283966b676 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Sat, 9 May 2020 20:52:42 +0200 Subject: [PATCH 193/962] chore: update changelog and prepare release 12.0.0 (#871) --- CHANGELOG.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 303b4adfc8..c5452ce16f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,27 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v12.0.0] - 2020-05-09 +BUG FIXES: +- Fix Launch Templates error with aws 2.61.0 ([#875](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/875)) +- Use splat syntax for cluster name to avoid `(known after apply)` in managed node groups ([#868](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/868)) + +DOCS: +- Add notes for Kubernetes 1.16 ([#873](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/873)) +- Remove useless template provider in examples ([#863](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/863)) + +FEATURES: +- Create kubeconfig with non-executable permissions ([#864](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/864)) +- Change EKS default version to 1.16 ([#857](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/857)) + +ENHANCEMENTS: +- Remove dependency on external template provider ([#854](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/854)) + +BREAKING CHANGES: +- The default `cluster_version` is now 1.16. Kubernetes 1.16 includes a number of deprecated API removals, and you need to ensure your applications and add ons are updated, or workloads could fail after the upgrade is complete. For more information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). For action you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). Please set explicitly your `cluster_version` to an older EKS version until your workloads are ready for Kubernetes 1.16. + + ## [v11.1.0] - 2020-04-23 BUG FIXES: @@ -50,6 +71,7 @@ TESTS: - remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.1.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.0.0...HEAD +[v12.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.1.0...v12.0.0 [v11.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.0.0...v11.1.0 [v11.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v10.0.0...v11.0.0 From 444343553cfc22474cfc1e3f4b00ace7031b2699 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Fri, 29 May 2020 17:29:08 +0200 Subject: [PATCH 194/962] docs: Update FAQ (#891) --- docs/faq.md | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index b57af69791..6f2506486d 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -28,13 +28,14 @@ Nodes need to be able to connect to other AWS services plus pull down container ### `aws-auth` ConfigMap not present -The module configures the `aws-auth` ConfigMap. This is used by the cluster to grant IAM users RBAC permissions in the cluster. Sometimes the map fails to apply correctly, especially if terraform could not access the cluster endpoint during cluster creation. +The module configures the `aws-auth` ConfigMap. This is used by the cluster to grant IAM users and roles RBAC permissions in the cluster, like the IAM role assigned to the worker nodes. -Confirm that the ConfigMap matches the contents of the generated `config-map-aws-auth_${cluster_name}.yaml` file. You can retrieve the live config by running the following in your terraform folder: +Confirm that the ConfigMap matches the contents of the `config_map_aws_auth` module output. You can retrieve the live config by running the following in your terraform folder: `kubectl --kubeconfig=kubeconfig_* -n kube-system get cm aws-auth -o yaml` -Apply the config with: -`kubectl --kubeconfig=kubeconfig_* apply -f config-map-aws-auth_*.yaml` +If the ConfigMap is missing or the contents are incorrect then ensure that you have properly configured the kubernetes provider block by referring to [README.md](https://github.com/terraform-aws-modules/terraform-aws-eks/#usage-example) and run `terraform apply` again. + +Users with `manage_aws_auth = false` will need to apply the ConfigMap themselves. ## How can I work with the cluster if I disable the public endpoint? @@ -42,10 +43,24 @@ You have to interact with the cluster from within the VPC that it's associated w Creating a new cluster with the public endpoint disabled is harder to achieve. You will either want to pass in a pre-configured cluster security group or apply the `aws-auth` configmap in a separate action. +## ConfigMap "aws-auth" already exists + +This can happen if the kubernetes provider has not been configured for use with the cluster. The kubernetes provider will be accessing your default kubernetes cluster which already has the map defined. Read [README.md](https://github.com/terraform-aws-modules/terraform-aws-eks/#usage-example) for more details on how to configure the kubernetes provider correctly. + +Users upgrading from modules before 8.0.0 will need to import their existing aws-auth ConfigMap in to the terraform state. See 8.0.0's [CHANGELOG](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/v8.0.0/CHANGELOG.md#v800---2019-12-11) for more details. + +## `Error: Get http://localhost/api/v1/namespaces/kube-system/configmaps/aws-auth: dial tcp 127.0.0.1:80: connect: connection refused` + +Usually this means that the kubernetes provider has not been configured, there is no default `~/.kube/config` and so the kubernetes provider is attempting to talk to localhost. + +You need to configure the kubernetes provider correctly. See [README.md](https://github.com/terraform-aws-modules/terraform-aws-eks/#usage-example) for more details. + ## How can I stop Terraform from removing the EKS tags from my VPC and subnets? You need to add the tags to the VPC and subnets yourself. See the [basic example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). +An alternative is to use the aws provider's [`ignore_tags` variable](https://www.terraform.io/docs/providers/aws/#ignore\_tags-configuration-block). However this can also cause terraform to display a perpetual difference. + ## How do I safely remove old worker groups? You've added new worker groups. Deleting worker groups from earlier in the list causes Terraform to want to recreate all worker groups. This is a limitation with how Terraform works and the module using `count` to create the ASGs and other resources. @@ -84,6 +99,8 @@ You are using the cluster autoscaler: Alternatively you can set the `asg_recreate_on_change = true` worker group option to get the ASG recreated after changes to the launch configuration or template. But be aware of the risks to cluster stability mentioned above. +You can also use a 3rd party tool like Gruntwork's kubergrunt. See the [`eks deploy`](https://github.com/gruntwork-io/kubergrunt#deploy) subcommand. + ## `aws_auth.tf: At 2:14: Unknown token: 2:14 IDENT` You are attempting to use a Terraform 0.12 module with Terraform 0.11. @@ -118,11 +135,11 @@ worker_groups = [ instance_type = "m5.large" platform = "linux" asg_desired_capacity = 2 - }, + }, ] ``` -2. Apply commands from https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support (use tab with name `Windows`) +2. Apply commands from https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support (use tab with name `Windows`) 3. Add one more worker group for Windows with required field `platform = "windows"` and update your cluster. Worker group example: @@ -143,4 +160,12 @@ worker_groups = [ ] ``` -4. Wtih `kubectl get nodes` you can see cluster with mixed (Linux/Windows) nodes support. +4. With `kubectl get nodes` you can see cluster with mixed (Linux/Windows) nodes support. + +## Worker nodes with labels do not join a 1.16+ cluster + +Kubelet restricts the allowed list of labels in the `kubernetes.io` namespace that can be applied to nodes starting in 1.16. + +Older configurations used labels like `kubernetes.io/lifecycle=spot` and this is no longer allowed. Use `node.kubernetes.io/lifecycle=spot` instead. + +Reference the `--node-labels` argument for your version of Kubenetes for the allowed prefixes. [Documentation for 1.16](https://v1-16.docs.kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) From c748899d0e512f4c27558a7ee8a48d426c788985 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Fri, 29 May 2020 17:45:24 +0200 Subject: [PATCH 195/962] cluster_id blocks on wait_for_cluster (#890) --- outputs.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/outputs.tf b/outputs.tf index 8ec702f359..ae91414759 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,6 +1,9 @@ output "cluster_id" { description = "The name/id of the EKS cluster." value = element(concat(aws_eks_cluster.this.*.id, list("")), 0) + # So that calling plans wait for the cluster to be available before attempting + # to use it. They will not need to duplicate this null_resource + depends_on = [null_resource.wait_for_cluster] } output "cluster_arn" { From 7ffe5fa88f154ed60b80445365b657c0a01f4a3d Mon Sep 17 00:00:00 2001 From: Brandon McNama Date: Sat, 30 May 2020 16:25:50 -0400 Subject: [PATCH 196/962] fix: Work around path bug in aws-iam-authenticator (#894) * fix: Work around path bug in aws-iam-authenticator `aws-iam-authenticator` has an open issue where it will not recognize IAM roles that include paths. This change causes the path supplied to `var.iam_path` to be stripped when generating the `aws-auth` ConfigMap in order to work around this. https://github.com/kubernetes-sigs/aws-iam-authenticator/issues/153 https://github.com/aws/containers-roadmap/issues/926 --- aws_auth.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aws_auth.tf b/aws_auth.tf index db6dea4708..b583c069a9 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -46,7 +46,9 @@ locals { module.node_groups.aws_auth_roles, ) : { - rolearn = role["worker_role_arn"] + # Work around https://github.com/kubernetes-sigs/aws-iam-authenticator/issues/153 + # Strip the leading slash off so that Terraform doesn't think it's a regex + rolearn = replace(role["worker_role_arn"], replace(var.iam_path, "/^//", ""), "") username = "system:node:{{EC2PrivateDNSName}}" groups = tolist(concat( [ From 3fefc2a66c6da76a2a7cce74a0b4ab9bdbc93048 Mon Sep 17 00:00:00 2001 From: Simon Gurcke Date: Sun, 31 May 2020 06:43:26 +1000 Subject: [PATCH 197/962] feat: Allow communication between pods on workers and pods using the primary cluster security group (optional) (#892) NOTES: New variable `worker_create_cluster_primary_security_group_rules` to allow communication between pods on workers and pods using the primary cluster security group (Managed Node Groups or Fargate). It defaults to `false` to avoid potential conflicts with existing security group rules users may have implemented. --- README.md | 1 + local.tf | 9 +++++---- outputs.tf | 2 +- variables.tf | 6 ++++++ workers.tf | 22 ++++++++++++++++++++++ 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f6f62991fc..c723b3e253 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | | worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"602401143452"` | no | | worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"801119661308"` | no | +| worker\_create\_cluster\_primary\_security\_group\_rules | Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group. | `bool` | `false` | no | | worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | `bool` | `false` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | `bool` | `true` | no | | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | diff --git a/local.tf b/local.tf index c72af075f6..2f07029cd6 100644 --- a/local.tf +++ b/local.tf @@ -8,10 +8,11 @@ locals { ) ] - cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id - cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name - cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) - worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id + cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id + cluster_primary_security_group_id = var.cluster_version >= 1.14 ? element(concat(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, list("")), 0) : null + cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name + cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) + worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] default_ami_id_linux = coalesce(local.workers_group_defaults.ami_id, data.aws_ami.eks_worker.id) diff --git a/outputs.tf b/outputs.tf index ae91414759..b89ffcc9cb 100644 --- a/outputs.tf +++ b/outputs.tf @@ -53,7 +53,7 @@ output "cluster_oidc_issuer_url" { output "cluster_primary_security_group_id" { description = "The cluster primary security group ID created by the EKS cluster on 1.14 or later. Referred to as 'Cluster security group' in the EKS console." - value = var.cluster_version >= 1.14 ? element(concat(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, list("")), 0) : null + value = local.cluster_primary_security_group_id } output "cloudwatch_log_group_name" { diff --git a/variables.tf b/variables.tf index b949524f05..460bdc687f 100644 --- a/variables.tf +++ b/variables.tf @@ -228,6 +228,12 @@ variable "worker_create_initial_lifecycle_hooks" { default = false } +variable "worker_create_cluster_primary_security_group_rules" { + description = "Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group." + type = bool + default = false +} + variable "permissions_boundary" { description = "If provided, all IAM roles will be created with this permissions boundary attached." type = string diff --git a/workers.tf b/workers.tf index a1908b7391..4b9265a3a9 100644 --- a/workers.tf +++ b/workers.tf @@ -338,6 +338,28 @@ resource "aws_security_group_rule" "workers_ingress_cluster_https" { type = "ingress" } +resource "aws_security_group_rule" "workers_ingress_cluster_primary" { + count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_eks ? 1 : 0 + description = "Allow pods running on workers to receive communication from cluster primary security group (e.g. Fargate pods)." + protocol = "all" + security_group_id = local.worker_security_group_id + source_security_group_id = local.cluster_primary_security_group_id + from_port = 0 + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "cluster_primary_ingress_workers" { + count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_eks ? 1 : 0 + description = "Allow pods running on workers to send communication to cluster primary security group (e.g. Fargate pods)." + protocol = "all" + security_group_id = local.cluster_primary_security_group_id + source_security_group_id = local.worker_security_group_id + from_port = 0 + to_port = 65535 + type = "ingress" +} + resource "aws_iam_role" "workers" { count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 name_prefix = var.workers_role_name != "" ? null : aws_eks_cluster.this[0].name From 05cd78593a2d7e1e9fe5cb591ba07e3a3bc3fbc9 Mon Sep 17 00:00:00 2001 From: graywolf-at-work <43144838+graywolf-at-work@users.noreply.github.com> Date: Tue, 2 Jun 2020 09:24:39 +0000 Subject: [PATCH 198/962] fix: Ensure kubeconfig ends with \n (#880) In order to be a text file according to POSIX, file needs to be composed of text lines. Text line is defined as sequence of characters ending in \n. Sadly, `~}` did strip everything including the \n, so the kubeconfing did not end with a new line. Output empty string at the end to make sure of it. --- templates/kubeconfig.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/kubeconfig.tpl b/templates/kubeconfig.tpl index ad1beec739..a99a0dfa8f 100644 --- a/templates/kubeconfig.tpl +++ b/templates/kubeconfig.tpl @@ -35,4 +35,4 @@ users: - name: ${k} value: ${v} %{~ endfor ~} -%{ endif ~} +%{ endif } From 7de18cd9cd882f6ad105ca375b13729537df9e68 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Fri, 5 Jun 2020 10:34:23 +0200 Subject: [PATCH 199/962] fix: Revert removal of templates provider (#883) Broke use case of passing in custom template content. Reverts most of the following PRs: - #865 - #863 - #854 --- README.md | 2 + data.tf | 92 ++++++++++++++++++++++++++++ examples/basic/main.tf | 4 ++ examples/irsa/main.tf | 4 ++ examples/launch_templates/main.tf | 4 ++ examples/managed_node_groups/main.tf | 4 ++ examples/secrets_encryption/main.tf | 4 ++ examples/spot_instances/main.tf | 4 ++ local.tf | 88 -------------------------- outputs.tf | 4 +- versions.tf | 1 + workers.tf | 2 +- workers_launch_template.tf | 2 +- 13 files changed, 123 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index c723b3e253..3977fd7186 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | local | >= 1.4 | | null | >= 2.1 | | random | >= 2.1 | +| template | >= 2.1 | ## Providers @@ -150,6 +151,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | local | >= 1.4 | | null | >= 2.1 | | random | >= 2.1 | +| template | >= 2.1 | ## Inputs diff --git a/data.tf b/data.tf index a574e83bc6..5909936031 100644 --- a/data.tf +++ b/data.tf @@ -55,6 +55,98 @@ data "aws_iam_policy_document" "cluster_assume_role_policy" { } } +data "template_file" "userdata" { + count = var.create_eks ? local.worker_group_count : 0 + template = lookup( + var.worker_groups[count.index], + "userdata_template_file", + file( + lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" + ? "${path.module}/templates/userdata_windows.tpl" + : "${path.module}/templates/userdata.sh.tpl" + ) + ) + + vars = merge({ + platform = lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) + cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] + cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] + pre_userdata = lookup( + var.worker_groups[count.index], + "pre_userdata", + local.workers_group_defaults["pre_userdata"], + ) + additional_userdata = lookup( + var.worker_groups[count.index], + "additional_userdata", + local.workers_group_defaults["additional_userdata"], + ) + bootstrap_extra_args = lookup( + var.worker_groups[count.index], + "bootstrap_extra_args", + local.workers_group_defaults["bootstrap_extra_args"], + ) + kubelet_extra_args = lookup( + var.worker_groups[count.index], + "kubelet_extra_args", + local.workers_group_defaults["kubelet_extra_args"], + ) + }, + lookup( + var.worker_groups[count.index], + "userdata_template_extra_args", + local.workers_group_defaults["userdata_template_extra_args"] + ) + ) +} + +data "template_file" "launch_template_userdata" { + count = var.create_eks ? local.worker_group_launch_template_count : 0 + template = lookup( + var.worker_groups_launch_template[count.index], + "userdata_template_file", + file( + lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" + ? "${path.module}/templates/userdata_windows.tpl" + : "${path.module}/templates/userdata.sh.tpl" + ) + ) + + vars = merge({ + platform = lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) + cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] + cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] + pre_userdata = lookup( + var.worker_groups_launch_template[count.index], + "pre_userdata", + local.workers_group_defaults["pre_userdata"], + ) + additional_userdata = lookup( + var.worker_groups_launch_template[count.index], + "additional_userdata", + local.workers_group_defaults["additional_userdata"], + ) + bootstrap_extra_args = lookup( + var.worker_groups_launch_template[count.index], + "bootstrap_extra_args", + local.workers_group_defaults["bootstrap_extra_args"], + ) + kubelet_extra_args = lookup( + var.worker_groups_launch_template[count.index], + "kubelet_extra_args", + local.workers_group_defaults["kubelet_extra_args"], + ) + }, + lookup( + var.worker_groups_launch_template[count.index], + "userdata_template_extra_args", + local.workers_group_defaults["userdata_template_extra_args"] + ) + ) +} + data "aws_iam_role" "custom_cluster_iam_role" { count = var.manage_cluster_iam_resources ? 0 : 1 name = var.cluster_iam_role_name diff --git a/examples/basic/main.tf b/examples/basic/main.tf index a21e530d2c..4ce49b95b9 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -19,6 +19,10 @@ provider "null" { version = "~> 2.1" } +provider "template" { + version = "~> 2.1" +} + data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index bbb9e8a5a3..849db9c28f 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -15,6 +15,10 @@ provider "null" { version = "~> 2.1" } +provider "template" { + version = "~> 2.1" +} + data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 8d9cff8246..d76a13c561 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -19,6 +19,10 @@ provider "null" { version = "~> 2.1" } +provider "template" { + version = "~> 2.1" +} + data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index 1e7f802fa7..eed19c24ff 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -19,6 +19,10 @@ provider "null" { version = "~> 2.1" } +provider "template" { + version = "~> 2.1" +} + data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index e4483539ca..1a639e1d20 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -19,6 +19,10 @@ provider "null" { version = "~> 2.1" } +provider "template" { + version = "~> 2.1" +} + data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 007cd60357..5406f1d234 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -19,6 +19,10 @@ provider "null" { version = "~> 2.1" } +provider "template" { + version = "~> 2.1" +} + data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } diff --git a/local.tf b/local.tf index 2f07029cd6..febf6f742d 100644 --- a/local.tf +++ b/local.tf @@ -147,92 +147,4 @@ locals { aws_authenticator_additional_args = var.kubeconfig_aws_authenticator_additional_args aws_authenticator_env_variables = var.kubeconfig_aws_authenticator_env_variables }) : "" - - userdata = [for worker in var.worker_groups : templatefile( - lookup( - worker, - "userdata_template_file", - lookup(worker, "platform", local.workers_group_defaults["platform"]) == "windows" - ? "${path.module}/templates/userdata_windows.tpl" - : "${path.module}/templates/userdata.sh.tpl" - ), - merge( - { - platform = lookup(worker, "platform", local.workers_group_defaults["platform"]) - cluster_name = aws_eks_cluster.this[0].name - endpoint = aws_eks_cluster.this[0].endpoint - cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data - pre_userdata = lookup( - worker, - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - worker, - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - worker, - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - worker, - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - worker, - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) - ) if var.create_eks - ] - - launch_template_userdata = [for worker in var.worker_groups_launch_template : templatefile( - lookup( - worker, - "userdata_template_file", - lookup(worker, "platform", local.workers_group_defaults["platform"]) == "windows" - ? "${path.module}/templates/userdata_windows.tpl" - : "${path.module}/templates/userdata.sh.tpl" - ), - merge( - { - platform = lookup(worker, "platform", local.workers_group_defaults["platform"]) - cluster_name = aws_eks_cluster.this[0].name - endpoint = aws_eks_cluster.this[0].endpoint - cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data - pre_userdata = lookup( - worker, - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - worker, - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - worker, - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - worker, - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - worker, - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) - ) if var.create_eks - ] } diff --git a/outputs.tf b/outputs.tf index b89ffcc9cb..e9628e6540 100644 --- a/outputs.tf +++ b/outputs.tf @@ -95,8 +95,8 @@ output "workers_asg_names" { output "workers_user_data" { description = "User data of worker groups" value = concat( - local.userdata, - local.launch_template_userdata, + data.template_file.userdata.*.rendered, + data.template_file.launch_template_userdata.*.rendered, ) } diff --git a/versions.tf b/versions.tf index 21e74d2e97..64f92a5d23 100644 --- a/versions.tf +++ b/versions.tf @@ -5,6 +5,7 @@ terraform { aws = ">= 2.52.0" local = ">= 1.4" null = ">= 2.1" + template = ">= 2.1" random = ">= 2.1" kubernetes = ">= 1.11.1" } diff --git a/workers.tf b/workers.tf index 4b9265a3a9..2e2a80f6d8 100644 --- a/workers.tf +++ b/workers.tf @@ -170,7 +170,7 @@ resource "aws_launch_configuration" "workers" { "key_name", local.workers_group_defaults["key_name"], ) - user_data_base64 = base64encode(local.userdata[count.index]) + user_data_base64 = base64encode(data.template_file.userdata.*.rendered[count.index]) ebs_optimized = lookup( var.worker_groups[count.index], "ebs_optimized", diff --git a/workers_launch_template.tf b/workers_launch_template.tf index bdf1efdfb0..69e4f465ca 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -262,7 +262,7 @@ resource "aws_launch_template" "workers_launch_template" { local.workers_group_defaults["key_name"], ) user_data = base64encode( - local.launch_template_userdata[count.index], + data.template_file.launch_template_userdata.*.rendered[count.index], ) ebs_optimized = lookup( From a89e91bf8f6a8f6bca6d2b778db0bdc7214c3952 Mon Sep 17 00:00:00 2001 From: Michael Marshall <47911938+michaeljmarshall@users.noreply.github.com> Date: Sat, 6 Jun 2020 02:55:35 -0600 Subject: [PATCH 200/962] feat: Add aws_security_group_rule.cluster_https_worker_ingress to output values (#901) --- README.md | 1 + outputs.tf | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 3977fd7186..fa565c847d 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | kubeconfig\_filename | The filename of the generated kubectl config. | | node\_groups | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys | | oidc\_provider\_arn | The ARN of the OIDC Provider if `enable_irsa = true`. | +| security\_group\_rule\_cluster\_https\_worker\_ingress | Security group rule responsible for allowing pods to communicate with the EKS cluster API. | | worker\_iam\_instance\_profile\_arns | default IAM instance profile ARN for EKS worker groups | | worker\_iam\_instance\_profile\_names | default IAM instance profile name for EKS worker groups | | worker\_iam\_role\_arn | default IAM role ARN for EKS worker groups | diff --git a/outputs.tf b/outputs.tf index e9628e6540..0e00989a83 100644 --- a/outputs.tf +++ b/outputs.tf @@ -165,3 +165,8 @@ output "node_groups" { description = "Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys" value = module.node_groups.node_groups } + +output "security_group_rule_cluster_https_worker_ingress" { + description = "Security group rule responsible for allowing pods to communicate with the EKS cluster API." + value = aws_security_group_rule.cluster_https_worker_ingress +} From f2c4383719354c5bfcb4f15d34555e6b1cc5406d Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Sat, 6 Jun 2020 11:46:37 +0200 Subject: [PATCH 201/962] chore: update changelog for release 12.1.0 (#910) --- CHANGELOG.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5452ce16f..9f1b920d63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,24 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v12.1.0] - 2020-06-06 +FEATURES: +- Add aws_security_group_rule.cluster_https_worker_ingress to output values ([#901](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/901)) +- Allow communication between pods on workers and pods using the primary cluster security group (optional) ([#892](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/892)) + +BUG FIXES: +- Revert removal of templates provider ([#883](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/883)) +- Ensure kubeconfig ends with \n ([#880](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/880)) +- Work around path bug in aws-iam-authenticator ([#894](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/894)) + +DOCS: +- Update FAQ ([#891](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/891)) + +NOTES: +- New variable `worker_create_cluster_primary_security_group_rules` to allow communication between pods on workers and pods using the primary cluster security group (Managed Node Groups or Fargate). It defaults to `false` to avoid potential conflicts with existing security group rules users may have implemented. + + ## [v12.0.0] - 2020-05-09 BUG FIXES: @@ -71,7 +89,8 @@ TESTS: - remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.0.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.1.0...HEAD +[v12.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.0.0...v12.1.0 [v12.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.1.0...v12.0.0 [v11.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.0.0...v11.1.0 [v11.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v10.0.0...v11.0.0 From 348f4412215efa125e6d90eb893afc4f883e6947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ariel=20Vi=C3=B1as?= Date: Wed, 10 Jun 2020 07:38:56 -0300 Subject: [PATCH 202/962] fix: remove unnecessary conditional in private access security group (#915) --- cluster.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.tf b/cluster.tf index 55a1747cfe..68a07369df 100644 --- a/cluster.tf +++ b/cluster.tf @@ -46,7 +46,7 @@ resource "aws_eks_cluster" "this" { } resource "aws_security_group_rule" "cluster_private_access" { - count = var.create_eks && var.manage_aws_auth && var.cluster_endpoint_private_access && var.cluster_endpoint_public_access == false ? 1 : 0 + count = var.create_eks && var.cluster_endpoint_private_access && var.cluster_endpoint_public_access == false ? 1 : 0 type = "ingress" from_port = 443 to_port = 443 From c4edc6fbcc4083b052c71e0b05f21308f18eaaff Mon Sep 17 00:00:00 2001 From: Tim Robinson Date: Wed, 24 Jun 2020 17:31:05 +1000 Subject: [PATCH 203/962] fix: Get `on_demand_allocation_strategy` from `local.workers_group_defaults` when deciding to use `mixed_instances_policy` (#908) --- workers_launch_template.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 69e4f465ca..2e0bd9759d 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -90,7 +90,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { dynamic mixed_instances_policy { iterator = item - for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", null) != null) ? list(var.worker_groups_launch_template[count.index]) : [] + for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? list(var.worker_groups_launch_template[count.index]) : [] content { instances_distribution { @@ -153,7 +153,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { } dynamic launch_template { iterator = item - for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", null) != null) ? [] : list(var.worker_groups_launch_template[count.index]) + for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? [] : list(var.worker_groups_launch_template[count.index]) content { id = aws_launch_template.workers_launch_template.*.id[count.index] From 60b494049b299cd985a1dec02ed285bc1e69b432 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Fri, 26 Jun 2020 17:39:49 +0200 Subject: [PATCH 204/962] docs: autoscaler X.Y version must match (#928) --- docs/autoscaling.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/autoscaling.md b/docs/autoscaling.md index d13d40941c..28d737f418 100644 --- a/docs/autoscaling.md +++ b/docs/autoscaling.md @@ -78,6 +78,10 @@ awsRegion: YOUR_AWS_REGION autoDiscovery: clusterName: YOUR_CLUSTER_NAME enabled: true + +image: + repository: us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler + tag: v1.16.5 ``` To install the chart, simply run helm with the `--values` option: @@ -89,3 +93,5 @@ helm install stable/cluster-autoscaler --values=path/to/your/values-file.yaml ## Notes There is a variable `asg_desired_capacity` given in the `local.tf` file, currently it can be used to change the desired worker(s) capacity in the autoscaling group but currently it is being ignored in terraform to reduce the [complexities](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/510#issuecomment-531700442) and the feature of scaling up and down the cluster nodes is being handled by the cluster autoscaler. + +The cluster autoscaler major and minor versions must match your cluster. For example if you are running a 1.16 EKS cluster set `image.tag=v1.16.5`. Search through their [releases page](https://github.com/kubernetes/autoscaler/releases) for valid version numbers. From 1e4a908a951bd1b819b2761736dabb8875d6d765 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Fri, 26 Jun 2020 17:40:34 +0200 Subject: [PATCH 205/962] docs: Improve FAQ on how to deploy from Windows (#927) --- docs/faq.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index 6f2506486d..043851d611 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -162,6 +162,18 @@ worker_groups = [ 4. With `kubectl get nodes` you can see cluster with mixed (Linux/Windows) nodes support. +## Deploying from Windows: `/bin/sh` file does not exist + +The module is almost pure Terraform apart from the `wait_for_cluster` `null_resource` that runs a local provisioner. The module has a default configuration for Unix-like systems. In order to run the provisioner on Windows systems you must set the interpreter to a valid value. [PR #795 (comment)](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/795#issuecomment-599191029) suggests the following value: +```hcl +module "eks" { + # ... + wait_for_cluster_interpreter = ["c:/git/bin/sh.exe", "-c"] +} +``` + +Alternatively, you can disable the `null_resource` by disabling creation of the `aws-auth` ConfigMap via setting `manage_aws_auth = false` on the module. The ConfigMap will then need creating via a different method. + ## Worker nodes with labels do not join a 1.16+ cluster Kubelet restricts the allowed list of labels in the `kubernetes.io` namespace that can be applied to nodes starting in 1.16. From 616d30ec674ff1d125710755f5073b1665bbd1af Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Sun, 28 Jun 2020 02:31:23 +0200 Subject: [PATCH 206/962] feat: Add a homemade `depends_on` for MNG submodule to ensure ordering of resource creation (#867) --- modules/node_groups/README.md | 1 + modules/node_groups/node_groups.tf | 2 ++ modules/node_groups/random.tf | 2 ++ modules/node_groups/variables.tf | 8 ++++++++ node_groups.tf | 33 +++++++++++++----------------- 5 files changed, 27 insertions(+), 19 deletions(-) diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index dce434906f..c905c749f2 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -52,6 +52,7 @@ No requirements. | cluster\_name | Name of parent cluster | `string` | n/a | yes | | create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | default\_iam\_role\_arn | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | n/a | yes | +| ng\_depends\_on | List of references to other resources this submodule depends on | `any` | `null` | no | | node\_groups | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | | node\_groups\_defaults | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | n/a | yes | | tags | A map of tags to add to all resources | `map(string)` | n/a | yes | diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 62dc6bff98..77fa02e4bc 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -47,4 +47,6 @@ resource "aws_eks_node_group" "workers" { create_before_destroy = true ignore_changes = [scaling_config.0.desired_size] } + + depends_on = [var.ng_depends_on] } diff --git a/modules/node_groups/random.tf b/modules/node_groups/random.tf index 14e7ba2bce..16c0583e1c 100644 --- a/modules/node_groups/random.tf +++ b/modules/node_groups/random.tf @@ -18,4 +18,6 @@ resource "random_pet" "node_groups" { subnet_ids = join("|", each.value["subnets"]) node_group_name = join("-", [var.cluster_name, each.key]) } + + depends_on = [var.ng_depends_on] } diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf index c0eaa23d1e..fc869d9d99 100644 --- a/modules/node_groups/variables.tf +++ b/modules/node_groups/variables.tf @@ -34,3 +34,11 @@ variable "node_groups" { type = any default = {} } + +# Hack for a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 +# Will be removed in Terraform 0.13 with the support of module's `depends_on` https://github.com/hashicorp/terraform/issues/10462 +variable "ng_depends_on" { + description = "List of references to other resources this submodule depends on" + type = any + default = null +} diff --git a/node_groups.tf b/node_groups.tf index 8ead35b89e..6721f51aa2 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -1,28 +1,23 @@ -# Hack to ensure ordering of resource creation. Do not create node_groups -# before other resources are ready. Removes race conditions -data "null_data_source" "node_groups" { - count = var.create_eks ? 1 : 0 - - inputs = { - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] - - # Ensure these resources are created before "unlocking" the data source. - # `depends_on` causes a refresh on every run so is useless here. - # [Re]creating or removing these resources will trigger recreation of Node Group resources - aws_auth = coalescelist(kubernetes_config_map.aws_auth[*].id, [""])[0] - role_NodePolicy = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy[*].id, [""])[0] - role_CNI_Policy = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy[*].id, [""])[0] - role_Container = coalescelist(aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly[*].id, [""])[0] - } -} - module "node_groups" { source = "./modules/node_groups" create_eks = var.create_eks - cluster_name = coalescelist(data.null_data_source.node_groups[*].outputs["cluster_name"], [""])[0] + cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] default_iam_role_arn = coalescelist(aws_iam_role.workers[*].arn, [""])[0] workers_group_defaults = local.workers_group_defaults tags = var.tags node_groups_defaults = var.node_groups_defaults node_groups = var.node_groups + + # Hack to ensure ordering of resource creation. + # This is a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 + # Do not create node_groups before other resources are ready and removes race conditions + # Ensure these resources are created before "unlocking" the data source. + # Will be removed in Terraform 0.13 + ng_depends_on = [ + aws_eks_cluster.this, + kubernetes_config_map.aws_auth, + aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy, + aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly + ] } From 9a0e548dcc9606ac79522363f26aacfc01647ff3 Mon Sep 17 00:00:00 2001 From: Ivan Sukhomlyn Date: Sun, 28 Jun 2020 15:44:36 +0300 Subject: [PATCH 207/962] feat: Add IAM permissions for ELB svc-linked role creation by EKS cluster (#902) AmazonEKSClusterPolicy IAM policy doesn't contain all necessary permissions to create ELB service-linked role required during LB provisioning at AWS by K8S Service. https://github.com/terraform-aws-modules/terraform-aws-eks/issues/900 https://github.com/terraform-aws-modules/terraform-aws-eks/issues/183#issuecomment-435229552 --- cluster.tf | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cluster.tf b/cluster.tf index 68a07369df..542ae1148e 100644 --- a/cluster.tf +++ b/cluster.tf @@ -130,3 +130,28 @@ resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { policy_arn = "${local.policy_arn_prefix}/AmazonEKSServicePolicy" role = local.cluster_iam_role_name } + +/* + Adding a policy to cluster IAM role that allow permissions + required to create AWSServiceRoleForElasticLoadBalancing service-linked role by EKS during ELB provisioning +*/ + +data "aws_iam_policy_document" "cluster_elb_sl_role_creation" { + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + + statement { + effect = "Allow" + actions = [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeInternetGateways" + ] + resources = ["*"] + } +} + +resource "aws_iam_role_policy" "cluster_elb_sl_role_creation" { + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + name_prefix = "${var.cluster_name}-elb-sl-role-creation" + role = local.cluster_iam_role_name + policy = data.aws_iam_policy_document.cluster_elb_sl_role_creation[0].json +} From db9bb0b48201b09b4c1df82b10c2aceac4981520 Mon Sep 17 00:00:00 2001 From: Marcus Vaal Date: Sun, 12 Jul 2020 10:14:36 -0500 Subject: [PATCH 208/962] feat: worker launch templates and configurations depend on security group rules and IAM policies (#933) In order to ensure proper ordering when running terraform destroy. This will block Terraform from removing up security group rules before the cluster has finished its clean up chores. --- cluster.tf | 2 ++ workers.tf | 22 +++++++++++++++++++--- workers_launch_template.tf | 16 ++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/cluster.tf b/cluster.tf index 542ae1148e..1a804702d0 100644 --- a/cluster.tf +++ b/cluster.tf @@ -39,6 +39,8 @@ resource "aws_eks_cluster" "this" { } depends_on = [ + aws_security_group_rule.cluster_egress_internet, + aws_security_group_rule.cluster_https_worker_ingress, aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy, aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy, aws_cloudwatch_log_group.this diff --git a/workers.tf b/workers.tf index 2e2a80f6d8..6c9d924f0e 100644 --- a/workers.tf +++ b/workers.tf @@ -256,6 +256,22 @@ resource "aws_launch_configuration" "workers" { lifecycle { create_before_destroy = true } + + # Prevent premature access of security group roles and policies by pods that + # require permissions on create/destroy that depend on workers. + depends_on = [ + aws_security_group_rule.workers_egress_internet, + aws_security_group_rule.workers_ingress_self, + aws_security_group_rule.workers_ingress_cluster, + aws_security_group_rule.workers_ingress_cluster_kubelet, + aws_security_group_rule.workers_ingress_cluster_https, + aws_security_group_rule.workers_ingress_cluster_primary, + aws_security_group_rule.cluster_primary_ingress_workers, + aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy, + aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.workers_additional_policies + ] } resource "random_pet" "workers" { @@ -271,14 +287,14 @@ resource "random_pet" "workers" { resource "aws_security_group" "workers" { count = var.worker_create_security_group && var.create_eks ? 1 : 0 - name_prefix = aws_eks_cluster.this[0].name + name_prefix = var.cluster_name description = "Security group for all nodes in the cluster." vpc_id = var.vpc_id tags = merge( var.tags, { - "Name" = "${aws_eks_cluster.this[0].name}-eks_worker_sg" - "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" = "owned" + "Name" = "${var.cluster_name}-eks_worker_sg" + "kubernetes.io/cluster/${var.cluster_name}" = "owned" }, ) } diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 2e0bd9759d..5ac64267a2 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -423,6 +423,22 @@ resource "aws_launch_template" "workers_launch_template" { lifecycle { create_before_destroy = true } + + # Prevent premature access of security group roles and policies by pods that + # require permissions on create/destroy that depend on workers. + depends_on = [ + aws_security_group_rule.workers_egress_internet, + aws_security_group_rule.workers_ingress_self, + aws_security_group_rule.workers_ingress_cluster, + aws_security_group_rule.workers_ingress_cluster_kubelet, + aws_security_group_rule.workers_ingress_cluster_https, + aws_security_group_rule.workers_ingress_cluster_primary, + aws_security_group_rule.cluster_primary_ingress_workers, + aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy, + aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.workers_additional_policies + ] } resource "random_pet" "workers_launch_template" { From b0ebef69b39e307d10fa41c4b293f2a9ba0aeba4 Mon Sep 17 00:00:00 2001 From: Michael Potter Date: Sun, 12 Jul 2020 08:40:27 -0700 Subject: [PATCH 209/962] fix: Strip user Name tag from asg_tags #946) --- local.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/local.tf b/local.tf index febf6f742d..13e4cd407e 100644 --- a/local.tf +++ b/local.tf @@ -6,6 +6,7 @@ locals { "value", element(values(var.tags), index(keys(var.tags), item)), "propagate_at_launch", "true" ) + if item != "Name" ] cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id From 23ad497557438b5fd95daaf33cf8859e091b9c98 Mon Sep 17 00:00:00 2001 From: Daniel Quackenbush <25692880+danquack@users.noreply.github.com> Date: Sun, 12 Jul 2020 12:25:48 -0400 Subject: [PATCH 210/962] feat: IMDSv2 metadata configuration in Launch Templates (#938) NOTES: Addition of the IMDSv2 metadata configuration block to Launch Templates will cause a diff to be generated for existing Launch Templates on first Terraform apply. The defaults match existing behaviour. --- README.md | 4 ++-- local.tf | 21 ++++++++++++--------- versions.tf | 2 +- workers_launch_template.tf | 18 ++++++++++++++++++ 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index fa565c847d..90b5eb44b2 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | terraform | >= 0.12.9 | -| aws | >= 2.52.0 | +| aws | >= 2.55.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | @@ -146,7 +146,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| aws | >= 2.52.0 | +| aws | >= 2.55.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | diff --git a/local.tf b/local.tf index 13e4cd407e..afc52bee21 100644 --- a/local.tf +++ b/local.tf @@ -76,15 +76,18 @@ locals { platform = "linux" # Platform of workers. either "linux" or "windows" additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults # Settings for launch templates - root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. - root_kms_key_id = "" # The KMS key to use when encrypting the root storage device - launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group - launch_template_placement_tenancy = "default" # The placement tenancy for instances - launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. - root_encrypted = false # Whether the volume should be encrypted or not - eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) - cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs - market_type = null + root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. + root_kms_key_id = "" # The KMS key to use when encrypting the root storage device + launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group + launch_template_placement_tenancy = "default" # The placement tenancy for instances + launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. + root_encrypted = false # Whether the volume should be encrypted or not + eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) + cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs + market_type = null + metadata_http_endpoint = "enabled" # The state of the metadata service: enabled, disabled. + metadata_http_tokens = "optional" # If session tokens are required: optional, required. + metadata_http_put_response_hop_limit = null # The desired HTTP PUT response hop limit for instance metadata requests. # Settings for launch templates with mixed instances policy override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] # A list of override instance types for mixed instances policy on_demand_allocation_strategy = null # Strategy to use when launching on-demand instances. Valid values: prioritized. diff --git a/versions.tf b/versions.tf index 64f92a5d23..f87acd8e5b 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.12.9" required_providers { - aws = ">= 2.52.0" + aws = ">= 2.55.0" local = ">= 1.4" null = ">= 2.1" template = ">= 2.1" diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 5ac64267a2..eaf734b41f 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -278,6 +278,24 @@ resource "aws_launch_template" "workers_launch_template" { ) ) + metadata_options { + http_endpoint = lookup( + var.worker_groups_launch_template[count.index], + "metadata_http_endpoint", + local.workers_group_defaults["metadata_http_endpoint"], + ) + http_tokens = lookup( + var.worker_groups_launch_template[count.index], + "metadata_http_tokens", + local.workers_group_defaults["metadata_http_tokens"], + ) + http_put_response_hop_limit = lookup( + var.worker_groups_launch_template[count.index], + "metadata_http_put_response_hop_limit", + local.workers_group_defaults["metadata_http_put_response_hop_limit"], + ) + } + credit_specification { cpu_credits = lookup( var.worker_groups_launch_template[count.index], From 7d86a6c75731c59ae8e439de2f3764dc73614591 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Mon, 13 Jul 2020 21:22:41 +0200 Subject: [PATCH 211/962] docs: Update required IAM permissions list (#936) NodeGroups, Fargate, EKS tagging, KMS --- docs/iam-permissions.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index 65eb3666c0..72d750244b 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -82,7 +82,21 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "eks:DescribeCluster", "eks:ListClusters", "eks:UpdateClusterConfig", + "eks:UpdateClusterVersion", "eks:DescribeUpdate", + "eks:TagResource", + "eks:UntagResource", + "eks:ListTagsForResource", + "eks:CreateFargateProfile", + "eks:DeleteFargateProfile", + "eks:DescribeFargateProfile", + "eks:ListFargateProfiles", + "eks:CreateNodegroup", + "eks:DeleteNodegroup", + "eks:DescribeNodegroup", + "eks:ListNodegroups", + "eks:UpdateNodegroupConfig", + "eks:UpdateNodegroupVersion", "iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy", "iam:CreateInstanceProfile", @@ -109,13 +123,22 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "iam:PutRolePolicy", "iam:RemoveRoleFromInstanceProfile", "iam:TagRole", + "iam:UntagRole", "iam:UpdateAssumeRolePolicy", // Following permissions are needed if cluster_enabled_log_types is enabled "logs:CreateLogGroup", "logs:DescribeLogGroups", "logs:DeleteLogGroup", "logs:ListTagsLogGroup", - "logs:PutRetentionPolicy" + "logs:PutRetentionPolicy", + // Following permissions for working with secrets_encryption example + "kms:CreateGrant", + "kms:CreateKey", + "kms:DescribeKey", + "kms:GetKeyPolicy", + "kms:GetKeyRotationStatus", + "kms:ListResourceTags", + "kms:ScheduleKeyDeletion" ], "Resource": "*" } From 3d2f7d28c132e62a91309eafd88c5d1d4500eb2b Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Mon, 13 Jul 2020 21:31:56 +0200 Subject: [PATCH 212/962] chore: update changelog for release 12.2.0 (#948) --- CHANGELOG.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f1b920d63..f74b1cfbd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,28 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v12.2.0] - 2020-07-12 +DOCS: +- Update required IAM permissions list ([#936](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/936)) +- Improve FAQ on how to deploy from Windows ([#927](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/927)) +- autoscaler X.Y version must match ([#928](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/928)) + +FEATURES: +- IMDSv2 metadata configuration in Launch Templates ([#938](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/938)) +- worker launch templates and configurations depend on security group rules and IAM policies ([#933](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/933)) +- Add IAM permissions for ELB svc-linked role creation by EKS cluster ([#902](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/902)) +- Add a homemade `depends_on` for MNG submodule to ensure ordering of resource creation ([#867](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/867)) + +BUG FIXES: +- Strip user Name tag from asg_tags [#946](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/946)) +- Get `on_demand_allocation_strategy` from `local.workers_group_defaults` when deciding to use `mixed_instances_policy` ([#908](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/908)) +- remove unnecessary conditional in private access security group ([#915](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/915)) + +NOTES: +- Addition of the IMDSv2 metadata configuration block to Launch Templates will cause a diff to be generated for existing Launch Templates on first Terraform apply. The defaults match existing behaviour. + + ## [v12.1.0] - 2020-06-06 FEATURES: @@ -89,7 +111,8 @@ TESTS: - remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.1.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.2.0...HEAD +[v12.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.1.0...v12.2.0 [v12.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.0.0...v12.1.0 [v12.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.1.0...v12.0.0 [v11.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.0.0...v11.1.0 From 9ace344c57709ac5cd32da2d94ed6b6f7e9c1d28 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Wed, 12 Aug 2020 15:22:24 +0200 Subject: [PATCH 213/962] docs: Better documentation of `cluster_id` output blocking (#955) NOTES: Starting in v12.1.0 the `cluster_id` output depends on the `wait_for_cluster` null resource. This means that initialisation of the kubernetes provider will be blocked until the cluster is really ready, if the module is set to manage the aws_auth ConfigMap and user followed the typical Usage Example. kubernetes resources in the same plan do not need to depend on anything explicitly. --- README.md | 2 +- docs/faq.md | 8 ++++++++ outputs.tf | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 90b5eb44b2..c607893f59 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_endpoint | The endpoint for your EKS Kubernetes API. | | cluster\_iam\_role\_arn | IAM role ARN of the EKS cluster. | | cluster\_iam\_role\_name | IAM role name of the EKS cluster. | -| cluster\_id | The name/id of the EKS cluster. | +| cluster\_id | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | cluster\_oidc\_issuer\_url | The URL on the EKS cluster OIDC Issuer | | cluster\_primary\_security\_group\_id | The cluster primary security group ID created by the EKS cluster on 1.14 or later. Referred to as 'Cluster security group' in the EKS console. | | cluster\_security\_group\_id | Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console. | diff --git a/docs/faq.md b/docs/faq.md index 043851d611..29cb2ee215 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -101,6 +101,14 @@ Alternatively you can set the `asg_recreate_on_change = true` worker group optio You can also use a 3rd party tool like Gruntwork's kubergrunt. See the [`eks deploy`](https://github.com/gruntwork-io/kubergrunt#deploy) subcommand. +## How do I create kubernetes resources when creating the cluster? + +You do not need to do anything extra since v12.1.0 of the module as long as the following conditions are met: +- `manage_aws_auth = true` on the module (default) +- the kubernetes provider is correctly configured like in the [Usage Example](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/README.md#usage-example). Primarily the module's `cluster_id` output is used as input to the `aws_eks_cluster*` data sources. + +The `cluster_id` depends on a `null_resource` that polls the EKS cluster's endpoint until it is alive. This blocks initialisation of the kubernetes provider. + ## `aws_auth.tf: At 2:14: Unknown token: 2:14 IDENT` You are attempting to use a Terraform 0.12 module with Terraform 0.11. diff --git a/outputs.tf b/outputs.tf index 0e00989a83..a1ac5e579b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,5 +1,5 @@ output "cluster_id" { - description = "The name/id of the EKS cluster." + description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = element(concat(aws_eks_cluster.this.*.id, list("")), 0) # So that calling plans wait for the cluster to be available before attempting # to use it. They will not need to duplicate this null_resource From 415cfca9af3c1f1219e3c294d85fe29a28b255a3 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Wed, 12 Aug 2020 16:10:31 +0200 Subject: [PATCH 214/962] ci: fix CI lint job (#973) Module used by an example does not allow use with 0.13 yet. Lock to latest version of 0.12. No 0.12 floating tag :( --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 57e27e7cc4..bfb0ebad63 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -33,7 +33,7 @@ jobs: name: Validate runs-on: ubuntu-latest container: - image: hashicorp/terraform:latest + image: hashicorp/terraform:0.12.29 steps: - uses: actions/checkout@master - name: Validate Code From f45b5a1d4614a19c0a0e790161ac552fc42f9d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20W?= Date: Fri, 14 Aug 2020 23:23:17 +1000 Subject: [PATCH 215/962] fix: random_pet with LT workers under 0.13.0 (#940) Fix for: Error: Provider produced inconsistent final plan When expanding the plan for module.eks.random_pet.workers_launch_template[0] to include new values learned so far during apply, provider "registry.terraform.io/hashicorp/random" changed the planned action from CreateThenDelete to DeleteThenCreate. --- .gitignore | 1 + workers_launch_template.tf | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 6b31084d05..db3aec7ca3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ eks-admin-service-account.yaml *.iml config-map-aws-auth*.yaml kubeconfig_* +*.swp diff --git a/workers_launch_template.tf b/workers_launch_template.tf index eaf734b41f..5c328c5d3e 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -476,6 +476,9 @@ resource "random_pet" "workers_launch_template" { ) ) } + lifecycle { + create_before_destroy = true + } } resource "aws_iam_instance_profile" "workers_launch_template" { From 1c8122704413fb3f54bff90b2692100def041caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giuseppe=20Crin=C3=B2?= Date: Fri, 14 Aug 2020 15:27:46 +0200 Subject: [PATCH 216/962] docs: Describe key_name is about AWS EC2 key pairs (#970) --- local.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local.tf b/local.tf index afc52bee21..98d7d2bdf6 100644 --- a/local.tf +++ b/local.tf @@ -52,7 +52,7 @@ locals { root_volume_size = "100" # root volume size of workers instances. root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp2', or 'io1' root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". - key_name = "" # The key name that should be used for the instances in the autoscaling group + key_name = "" # The key pair name that should be used for the instances in the autoscaling group pre_userdata = "" # userdata to pre-append to the default userdata. userdata_template_file = "" # alternate template to use for userdata userdata_template_extra_args = {} # Additional arguments to use when expanding the userdata template file From c508d12b696a24da87f72faa9d3ec073e9ff830b Mon Sep 17 00:00:00 2001 From: Thomas O'Neill Date: Fri, 14 Aug 2020 09:58:34 -0400 Subject: [PATCH 217/962] docs: Mixed spot + on-demand instance documentation (#967) * Add example for lauch config with mixed lifecycles * Set what on-demand instance is * Tweak wording Co-authored-by: Thomas O'Neill Co-authored-by: Daniel Piddock --- docs/spot-instances.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/spot-instances.md b/docs/spot-instances.md index 1eb4da4257..442c228ebc 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -83,6 +83,30 @@ Launch Template support is a recent addition to both AWS and this module. It mig ] ``` +## Using Launch Templates With Both Spot and On Demand + +Example launch template to launch 2 on demand instances of type m5.large, and have the ability to scale up using spot instances and on demand instances. The `node.kubernetes.io/lifecycle` node label will be set to the value queried from the EC2 meta-data service: either "on-demand" or "spot". + +`on_demand_percentage_above_base_capacity` is set to 25 so 1 in 4 new nodes, when auto-scaling, will be on-demand instances. If not set, all new nodes will be spot instances. The on-demand instances will be the primary instance type (first in the array if they are not weighted). + +```hcl + worker_groups_launch_template = [{ + name = "mixed-demand-spot" + override_instance_types = ["m5.large", "m5a.large", "m4.large"] + root_encrypted = true + root_volume_size = 50 + + asg_min_size = 2 + asg_desired_capacity = 2 + on_demand_base_capacity = 3 + on_demand_percentage_above_base_capacity = 25 + asg_max_size = 20 + spot_instance_pools = 3 + + kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=`curl -s http://169.254.169.254/latest/meta-data/instance-life-cycle`" + }] +``` + ## Important Notes An issue with the cluster-autoscaler: https://github.com/kubernetes/autoscaler/issues/1133 From 1bd5bf5ab9e4f2701c064668a02f012d971a1102 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Fri, 14 Aug 2020 16:06:53 +0200 Subject: [PATCH 218/962] feat: Cluster version is now a required variable. (#972) BREAKING CHANGES: `cluster_version` variable is now required. --- README.md | 2 +- examples/basic/main.tf | 7 ++++--- examples/create_false/main.tf | 5 +++-- examples/irsa/main.tf | 11 ++++++----- examples/launch_templates/main.tf | 9 +++++---- examples/managed_node_groups/main.tf | 7 ++++--- examples/secrets_encryption/main.tf | 7 ++++--- examples/spot_instances/main.tf | 9 +++++---- variables.tf | 1 - 9 files changed, 32 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index c607893f59..90180f0d49 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | -| cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | `"1.16"` | no | +| cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | | create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | eks\_oidc\_root\_ca\_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 4ce49b95b9..8c86e97ac6 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -123,9 +123,10 @@ module "vpc" { } module "eks" { - source = "../.." - cluster_name = local.cluster_name - subnets = module.vpc.private_subnets + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.private_subnets tags = { Environment = "test" diff --git a/examples/create_false/main.tf b/examples/create_false/main.tf index 86604aacdd..0aa61ef959 100644 --- a/examples/create_false/main.tf +++ b/examples/create_false/main.tf @@ -21,8 +21,9 @@ provider "kubernetes" { } module "eks" { - source = "../.." - create_eks = false + source = "../.." + create_eks = false + cluster_version = "" vpc_id = "" cluster_name = "" diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index 849db9c28f..2c26bd1b82 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -55,11 +55,12 @@ module "vpc" { } module "eks" { - source = "../.." - cluster_name = local.cluster_name - subnets = module.vpc.public_subnets - vpc_id = module.vpc.vpc_id - enable_irsa = true + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.public_subnets + vpc_id = module.vpc.vpc_id + enable_irsa = true worker_groups = [ { diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index d76a13c561..01db32dc5a 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -63,10 +63,11 @@ module "vpc" { } module "eks" { - source = "../.." - cluster_name = local.cluster_name - subnets = module.vpc.public_subnets - vpc_id = module.vpc.vpc_id + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.public_subnets + vpc_id = module.vpc.vpc_id worker_groups_launch_template = [ { diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index eed19c24ff..a9cbbb6b52 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -76,9 +76,10 @@ module "vpc" { } module "eks" { - source = "../.." - cluster_name = local.cluster_name - subnets = module.vpc.private_subnets + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.private_subnets tags = { Environment = "test" diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index 1a639e1d20..acb233ec50 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -80,9 +80,10 @@ module "vpc" { } module "eks" { - source = "../.." - cluster_name = local.cluster_name - subnets = module.vpc.private_subnets + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.private_subnets cluster_encryption_config = [ { diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 5406f1d234..c52a1fc79c 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -63,10 +63,11 @@ module "vpc" { } module "eks" { - source = "../.." - cluster_name = local.cluster_name - subnets = module.vpc.public_subnets - vpc_id = module.vpc.vpc_id + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.public_subnets + vpc_id = module.vpc.vpc_id worker_groups_launch_template = [ { diff --git a/variables.tf b/variables.tf index 460bdc687f..9a5e776452 100644 --- a/variables.tf +++ b/variables.tf @@ -28,7 +28,6 @@ variable "cluster_security_group_id" { variable "cluster_version" { description = "Kubernetes version to use for the EKS cluster." type = string - default = "1.16" } variable "config_output_path" { From 44d59da57b8298b99986179de6c725af05d85086 Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Thu, 27 Aug 2020 10:08:11 +0200 Subject: [PATCH 219/962] docs: Update README about `cluster_version` variable requirement (#988) --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 90180f0d49..89f016ace0 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,14 @@ Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aw ## Important note -The default `cluster_version`is now 1.16. Kubernetes 1.16 includes a number of deprecated API removals, and you need to ensure your applications and add ons are updated, or workloads could fail after the upgrade is complete. For more information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). For action you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html#1-16-prequisites). +The `cluster_version` is the required variable. Kubernetes is evolving a lot, and each major version includes new features, fixes, or changes. -Please set explicitly your `cluster_version` to an older EKS version until your workloads are ready for Kubernetes 1.16. +**Always check [Kubernetes Release Notes](https://kubernetes.io/docs/setup/release/notes/) before updating the major version.** + + +You also need to ensure your applications and add ons are updated, or workloads could fail after the upgrade is complete. For action, you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). + +An example of harming update was the removal of several commonly used, but deprecated APIs, in Kubernetes 1.16. More information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). ## Usage example @@ -45,7 +50,7 @@ provider "kubernetes" { module "my-cluster" { source = "terraform-aws-modules/eks/aws" cluster_name = "my-cluster" - cluster_version = "1.16" + cluster_version = "1.17" subnets = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] vpc_id = "vpc-1234556abcdef" From 1adbe82e1564ecaddee393d329a5b4f99393b7da Mon Sep 17 00:00:00 2001 From: Anthony Dahanne Date: Wed, 2 Sep 2020 16:38:02 -0400 Subject: [PATCH 220/962] fix: Create `cluster_private_access` security group rules when it should (#981) BREAKING CHANGES: Default for `cluster_endpoint_private_access_cidrs` is now `null` instead of `["0.0.0.0/0"]`. It makes the variable required when `cluster_create_endpoint_private_access_sg_rule` is set to `true`. This will force everyone who want to have a private access to set explicitly their allowed subnets for the sake of the principle of least access by default. --- README.md | 3 ++- cluster.tf | 2 +- variables.tf | 10 ++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 89f016ace0..5613d11e57 100644 --- a/README.md +++ b/README.md @@ -163,13 +163,14 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | +| cluster\_create\_endpoint\_private\_access\_sg\_rule | Whether to create security group rules for the access to the Amazon EKS private API server endpoint. | `bool` | `false` | no | | cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | | cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | | cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | | cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | | cluster\_encryption\_config | Configuration block with encryption configuration for the cluster. See examples/secrets\_encryption/main.tf for example format |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | | cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | -| cluster\_endpoint\_private\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS private API server endpoint, when public access is disabled | `list(string)` |
[
"0.0.0.0/0"
]
| no | +| cluster\_endpoint\_private\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS private API server endpoint. | `list(string)` | `null` | no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | | cluster\_endpoint\_public\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | | cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. | `string` | `""` | no | diff --git a/cluster.tf b/cluster.tf index 1a804702d0..0df121f258 100644 --- a/cluster.tf +++ b/cluster.tf @@ -48,7 +48,7 @@ resource "aws_eks_cluster" "this" { } resource "aws_security_group_rule" "cluster_private_access" { - count = var.create_eks && var.cluster_endpoint_private_access && var.cluster_endpoint_public_access == false ? 1 : 0 + count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access ? 1 : 0 type = "ingress" from_port = 443 to_port = 443 diff --git a/variables.tf b/variables.tf index 9a5e776452..c14976a9bc 100644 --- a/variables.tf +++ b/variables.tf @@ -245,10 +245,16 @@ variable "iam_path" { default = "/" } +variable "cluster_create_endpoint_private_access_sg_rule" { + description = "Whether to create security group rules for the access to the Amazon EKS private API server endpoint." + type = bool + default = false +} + variable "cluster_endpoint_private_access_cidrs" { - description = "List of CIDR blocks which can access the Amazon EKS private API server endpoint, when public access is disabled" + description = "List of CIDR blocks which can access the Amazon EKS private API server endpoint." type = list(string) - default = ["0.0.0.0/0"] + default = null } variable "cluster_endpoint_private_access" { From ad5373dc614c3d4d4e70d7177b25f6f084b67197 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 4 Sep 2020 23:04:41 +0200 Subject: [PATCH 221/962] docs: Add important notes about the retry logic and the `wget` requirement (#999) --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5613d11e57..956cb863c1 100644 --- a/README.md +++ b/README.md @@ -17,14 +17,19 @@ Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aw ## Important note -The `cluster_version` is the required variable. Kubernetes is evolving a lot, and each major version includes new features, fixes, or changes. - -**Always check [Kubernetes Release Notes](https://kubernetes.io/docs/setup/release/notes/) before updating the major version.** +The `cluster_version` is the required variable. Kubernetes is evolving a lot, and each major version includes new features, fixes, or changes. +**Always check [Kubernetes Release Notes](https://kubernetes.io/docs/setup/release/notes/) before updating the major version.** You also need to ensure your applications and add ons are updated, or workloads could fail after the upgrade is complete. For action, you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). -An example of harming update was the removal of several commonly used, but deprecated APIs, in Kubernetes 1.16. More information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). +An example of harming update was the removal of several commonly used, but deprecated APIs, in Kubernetes 1.16. More information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). + +By default, this module manage the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready, we implemented a retry logic with an `local-exec` provisioner and `wget` (by default). + +**If you want to manage your `aws-auth` configmap, ensure you have `wget` and `/bin/sh` installed where you're running Terraform or set `wait_for_cluster_cmd` and `wait_for_cluster_interpreter` to match your needs.** + +For windows users, please read the following [doc](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#deploying-from-windows-binsh-file-does-not-exist). ## Usage example From bceb966dfd6f763ad8ea58764e8e84922848eca4 Mon Sep 17 00:00:00 2001 From: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> Date: Tue, 8 Sep 2020 18:25:46 +0200 Subject: [PATCH 222/962] fix: More compatibility fixes for Terraform v0.13 and aws v3 (#976) Additional support for Terraform v0.13 and aws v3! - The update to the vpc module in examples was, strictly speaking, unnecessary but it adds the terraform block with supported versions. - Update for iam module in the example was very necessary to support new versions - Workaround for "Provider produced inconsistent final plan" when creating ASGs at the same time as the cluster. See https://github.com/terraform-providers/terraform-provider-aws/issues/14085 for full details. - Blacklist 0.13.0 as it was too strict when migrating from aws v2 to v3 about dropped attributes. --- README.md | 2 +- examples/basic/main.tf | 2 +- examples/irsa/irsa.tf | 2 +- examples/irsa/main.tf | 2 +- examples/launch_templates/main.tf | 2 +- examples/managed_node_groups/main.tf | 2 +- examples/secrets_encryption/main.tf | 2 +- examples/spot_instances/main.tf | 2 +- versions.tf | 2 +- workers.tf | 55 ++++++++++++++++------------ workers_launch_template.tf | 54 +++++++++++++++------------ 11 files changed, 71 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 956cb863c1..b0236d1ee6 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| terraform | >= 0.12.9 | +| terraform | >= 0.12.9, != 0.13.0 | | aws | >= 2.55.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 8c86e97ac6..e5158af922 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -100,7 +100,7 @@ resource "aws_security_group" "all_worker_mgmt" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.6.0" + version = "2.47.0" name = "test-vpc" cidr = "10.0.0.0/16" diff --git a/examples/irsa/irsa.tf b/examples/irsa/irsa.tf index 8ba8f06823..d219c3cfc1 100644 --- a/examples/irsa/irsa.tf +++ b/examples/irsa/irsa.tf @@ -1,6 +1,6 @@ module "iam_assumable_role_admin" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "~> v2.6.0" + version = "2.14.0" create_role = true role_name = "cluster-autoscaler" provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index 2c26bd1b82..49992af7a9 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -41,7 +41,7 @@ data "aws_caller_identity" "current" {} module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.6.0" + version = "2.47.0" name = "test-vpc" cidr = "10.0.0.0/16" azs = data.aws_availability_zones.available.names diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 01db32dc5a..40558d4d85 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -53,7 +53,7 @@ resource "random_string" "suffix" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.6.0" + version = "2.47.0" name = "test-vpc-lt" cidr = "10.0.0.0/16" diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index a9cbbb6b52..6d66aa21ea 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -53,7 +53,7 @@ resource "random_string" "suffix" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 2.6" + version = "2.47.0" name = "test-vpc" cidr = "172.16.0.0/16" diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index acb233ec50..7462980ea7 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -57,7 +57,7 @@ resource "aws_kms_key" "eks" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.6.0" + version = "2.47.0" name = "test-vpc" cidr = "10.0.0.0/16" diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index c52a1fc79c..d2f77f76d7 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -53,7 +53,7 @@ resource "random_string" "suffix" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.6.0" + version = "2.47.0" name = "test-vpc-spot" cidr = "10.0.0.0/16" diff --git a/versions.tf b/versions.tf index f87acd8e5b..27c659f933 100644 --- a/versions.tf +++ b/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.12.9" + required_version = ">= 0.12.9, != 0.13.0" required_providers { aws = ">= 2.55.0" diff --git a/workers.tf b/workers.tf index 6c9d924f0e..a5b6672951 100644 --- a/workers.tf +++ b/workers.tf @@ -102,31 +102,38 @@ resource "aws_autoscaling_group" "workers" { } } - tags = concat( - [ - { - "key" = "Name" - "value" = "${aws_eks_cluster.this[0].name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg" - "propagate_at_launch" = true - }, - { - "key" = "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" - "value" = "owned" - "propagate_at_launch" = true - }, - { - "key" = "k8s.io/cluster/${aws_eks_cluster.this[0].name}" - "value" = "owned" - "propagate_at_launch" = true - }, - ], - local.asg_tags, - lookup( - var.worker_groups[count.index], - "tags", - local.workers_group_defaults["tags"] + dynamic "tag" { + for_each = concat( + [ + { + "key" = "Name" + "value" = "${aws_eks_cluster.this[0].name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg" + "propagate_at_launch" = true + }, + { + "key" = "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" + "value" = "owned" + "propagate_at_launch" = true + }, + { + "key" = "k8s.io/cluster/${aws_eks_cluster.this[0].name}" + "value" = "owned" + "propagate_at_launch" = true + }, + ], + local.asg_tags, + lookup( + var.worker_groups[count.index], + "tags", + local.workers_group_defaults["tags"] + ) ) - ) + content { + key = tag.value.key + value = tag.value.value + propagate_at_launch = tag.value.propagate_at_launch + } + } lifecycle { create_before_destroy = true diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 5c328c5d3e..600666a11d 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -151,6 +151,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { } } } + dynamic launch_template { iterator = item for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? [] : list(var.worker_groups_launch_template[count.index]) @@ -178,30 +179,37 @@ resource "aws_autoscaling_group" "workers_launch_template" { } } - tags = concat( - [ - { - "key" = "Name" - "value" = "${aws_eks_cluster.this[0].name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}-eks_asg" - "propagate_at_launch" = true - }, - { - "key" = "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" - "value" = "owned" - "propagate_at_launch" = true - }, - ], - local.asg_tags, - lookup( - var.worker_groups_launch_template[count.index], - "tags", - local.workers_group_defaults["tags"] + dynamic "tag" { + for_each = concat( + [ + { + "key" = "Name" + "value" = "${aws_eks_cluster.this[0].name}-${lookup( + var.worker_groups_launch_template[count.index], + "name", + count.index, + )}-eks_asg" + "propagate_at_launch" = true + }, + { + "key" = "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" + "value" = "owned" + "propagate_at_launch" = true + }, + ], + local.asg_tags, + lookup( + var.worker_groups_launch_template[count.index], + "tags", + local.workers_group_defaults["tags"] + ) ) - ) + content { + key = tag.value.key + value = tag.value.value + propagate_at_launch = tag.value.propagate_at_launch + } + } lifecycle { create_before_destroy = true From 31ad394dbc61390dc46643b571249a2b670e9caa Mon Sep 17 00:00:00 2001 From: Thomas O'Neill Date: Sun, 4 Oct 2020 14:08:02 -0400 Subject: [PATCH 223/962] feat: Add AmazonEKSVPCResourceController to cluster policy to be able to set AWS Security Groups for pod (#1011) Co-authored-by: Thomas O'Neill --- cluster.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cluster.tf b/cluster.tf index 0df121f258..1693926247 100644 --- a/cluster.tf +++ b/cluster.tf @@ -43,6 +43,7 @@ resource "aws_eks_cluster" "this" { aws_security_group_rule.cluster_https_worker_ingress, aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy, aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy, + aws_iam_role_policy_attachment.cluster_AmazonEKSVPCResourceControllerPolicy, aws_cloudwatch_log_group.this ] } @@ -133,6 +134,12 @@ resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { role = local.cluster_iam_role_name } +resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSVPCResourceControllerPolicy" { + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEKSVPCResourceController" + role = local.cluster_iam_role_name +} + /* Adding a policy to cluster IAM role that allow permissions required to create AWSServiceRoleForElasticLoadBalancing service-linked role by EKS during ELB provisioning From 594602f75a88939471fe978f9afa106089628016 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Sun, 4 Oct 2020 20:45:43 +0200 Subject: [PATCH 224/962] ci: Bump terraform pre-commit hook version and re-run terraform-docs with the latest version to fix the CI (#1033) * bump pre-commit hook version * re-run terraform-docs with the latest version --- .pre-commit-config.yaml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e6874c6a0c..4949b1cbb5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.30.0 + rev: v1.43.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index b0236d1ee6..b932e49918 100644 --- a/README.md +++ b/README.md @@ -206,7 +206,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | -| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do wget --no-check-certificate -O - -q $ENDPOINT/healthz \u003e/dev/null \u0026\u0026 exit 0 || true; sleep 5; done; echo TIMEOUT \u0026\u0026 exit 1"` | no | +| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; sleep 5; done; echo TIMEOUT && exit 1"` | no | | wait\_for\_cluster\_interpreter | Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy. | `list(string)` |
[
"/bin/sh",
"-c"
]
| no | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | From f0ae6c67515f276f8043a1787c68a2ab61efa414 Mon Sep 17 00:00:00 2001 From: Lorenzo Aiello Date: Sun, 4 Oct 2020 11:49:41 -0700 Subject: [PATCH 225/962] feat: Dynamic Partition for IRSA to support AWS-CN Deployments (#1028) --- irsa.tf | 2 +- local.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/irsa.tf b/irsa.tf index 11c0cc735c..63a19871b5 100644 --- a/irsa.tf +++ b/irsa.tf @@ -9,7 +9,7 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { count = var.enable_irsa && var.create_eks ? 1 : 0 - client_id_list = ["sts.amazonaws.com"] + client_id_list = [local.sts_principal] thumbprint_list = [var.eks_oidc_root_ca_thumbprint] url = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0] } diff --git a/local.tf b/local.tf index 98d7d2bdf6..9b9807ec32 100644 --- a/local.tf +++ b/local.tf @@ -31,6 +31,7 @@ locals { ) ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" + sts_principal = "sts.${data.aws_partition.current.dns_suffix}" policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" workers_group_defaults_defaults = { From c4e95b3b04952b55054de2bdcfd43634573e501a Mon Sep 17 00:00:00 2001 From: Abhinav Sonkar Date: Sun, 4 Oct 2020 21:19:05 +0200 Subject: [PATCH 226/962] docs: Add `ec2:CreateVpcEndpoint` action to minimum permission list (#1022) --- docs/iam-permissions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index 72d750244b..c175b16429 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -42,6 +42,7 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "ec2:CreateTags", "ec2:CreateVolume", "ec2:CreateVpc", + "ec2:CreateVpcEndpoint", "ec2:DeleteDhcpOptions", "ec2:DeleteEgressOnlyInternetGateway", "ec2:DeleteInternetGateway", From d8ab5d4d6d86622859846cf7775ff9aba15bf35f Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Mon, 5 Oct 2020 14:59:40 +0200 Subject: [PATCH 227/962] improvement: update the `wait_for_cluster_cmd` logic to use `curl` if `wget` doesn't exist (#1002) --- README.md | 6 +++--- variables.tf | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b932e49918..efd6681fbc 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ You also need to ensure your applications and add ons are updated, or workloads An example of harming update was the removal of several commonly used, but deprecated APIs, in Kubernetes 1.16. More information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). -By default, this module manage the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready, we implemented a retry logic with an `local-exec` provisioner and `wget` (by default). +By default, this module manage the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready, we implemented a retry logic with an `local-exec` provisioner and `wget` (by default) with failover to `curl`. -**If you want to manage your `aws-auth` configmap, ensure you have `wget` and `/bin/sh` installed where you're running Terraform or set `wait_for_cluster_cmd` and `wait_for_cluster_interpreter` to match your needs.** +**If you want to manage your `aws-auth` configmap, ensure you have `wget` (or `curl`) and `/bin/sh` installed where you're running Terraform or set `wait_for_cluster_cmd` and `wait_for_cluster_interpreter` to match your needs.** For windows users, please read the following [doc](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#deploying-from-windows-binsh-file-does-not-exist). @@ -206,7 +206,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | -| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; sleep 5; done; echo TIMEOUT && exit 1"` | no | +| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do if `command -v wget > /dev/null`; then wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; else curl -k -s $ENDPOINT/healthz >/dev/null && exit 0 || true;fi; sleep 5; done; echo TIMEOUT && exit 1"` | no | | wait\_for\_cluster\_interpreter | Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy. | `list(string)` |
[
"/bin/sh",
"-c"
]
| no | | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | diff --git a/variables.tf b/variables.tf index c14976a9bc..f17bbd2b3a 100644 --- a/variables.tf +++ b/variables.tf @@ -200,7 +200,7 @@ variable "cluster_delete_timeout" { variable "wait_for_cluster_cmd" { description = "Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT" type = string - default = "for i in `seq 1 60`; do wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; sleep 5; done; echo TIMEOUT && exit 1" + default = "for i in `seq 1 60`; do if `command -v wget > /dev/null`; then wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; else curl -k -s $ENDPOINT/healthz >/dev/null && exit 0 || true;fi; sleep 5; done; echo TIMEOUT && exit 1" } variable "wait_for_cluster_interpreter" { From 8c8f4b56eb27430a6c5a48507de681914123612c Mon Sep 17 00:00:00 2001 From: hacker65536 Date: Mon, 5 Oct 2020 22:07:57 +0900 Subject: [PATCH 228/962] feat: Add `load_balancers` parameter to associate a CLB (Classic Load Balancer) to worker groups ASG (#992) --- local.tf | 1 + workers.tf | 5 +++++ workers_launch_template.tf | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/local.tf b/local.tf index 9b9807ec32..393dd7a857 100644 --- a/local.tf +++ b/local.tf @@ -70,6 +70,7 @@ locals { iam_role_id = "local.default_iam_role_id" # A custom IAM role id. Incompatible with iam_instance_profile_name. Literal local.default_iam_role_id will never be used but if iam_role_id is not set, the local.default_iam_role_id interpolation will be used. suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] target_group_arns = null # A list of Application LoadBalancer (ALB) target group ARNs to be associated to the autoscaling group + load_balancers = null # A list of Classic LoadBalancer (CLB)'s name to be associated to the autoscaling group enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] placement_group = null # The name of the placement group into which to launch the instances, if any. service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS diff --git a/workers.tf b/workers.tf index a5b6672951..c6618a8451 100644 --- a/workers.tf +++ b/workers.tf @@ -37,6 +37,11 @@ resource "aws_autoscaling_group" "workers" { "target_group_arns", local.workers_group_defaults["target_group_arns"] ) + load_balancers = lookup( + var.worker_groups[count.index], + "load_balancers", + local.workers_group_defaults["load_balancers"] + ) service_linked_role_arn = lookup( var.worker_groups[count.index], "service_linked_role_arn", diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 600666a11d..7e779b3b3a 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -37,6 +37,11 @@ resource "aws_autoscaling_group" "workers_launch_template" { "target_group_arns", local.workers_group_defaults["target_group_arns"] ) + load_balancers = lookup( + var.worker_groups_launch_template[count.index], + "load_balancers", + local.workers_group_defaults["load_balancers"] + ) service_linked_role_arn = lookup( var.worker_groups_launch_template[count.index], "service_linked_role_arn", From 93614b329d1f2584b94e44f77328388663712093 Mon Sep 17 00:00:00 2001 From: Guillaume GILL Date: Mon, 5 Oct 2020 15:24:47 +0200 Subject: [PATCH 229/962] improvement: Make the `cpu_credits` optional for workers launch template (#1030) NOTES: `credit_specification` for worker groups launch template can now be set to `null` so that we can use non burstable EC2 families --- workers_launch_template.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 7e779b3b3a..2546393dc8 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -309,12 +309,15 @@ resource "aws_launch_template" "workers_launch_template" { ) } - credit_specification { - cpu_credits = lookup( + dynamic "credit_specification" { + for_each = lookup( var.worker_groups_launch_template[count.index], "cpu_credits", local.workers_group_defaults["cpu_credits"] - ) + ) != null ? [lookup(var.worker_groups_launch_template[count.index], "cpu_credits", local.workers_group_defaults["cpu_credits"])] : [] + content { + cpu_credits = credit_specification.value + } } monitoring { From c799fecd85ea1879305f67fcb4106d13119e1a79 Mon Sep 17 00:00:00 2001 From: Rich Lafferty Date: Tue, 6 Oct 2020 03:24:22 -0400 Subject: [PATCH 230/962] fix: Use customer managed policy instead of inline policy for `cluster_elb_sl_role_creation` (#1039) NOTE: The usage of customer managed policy, not an inline policy, for the `cluster_elb_sl_role_creation policy` is common for "enterprise" AWS users to disallow inline policies with an SCP rule for auditing-related reasons, and this accomplishes the same thing. --- cluster.tf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cluster.tf b/cluster.tf index 1693926247..34b7314b5c 100644 --- a/cluster.tf +++ b/cluster.tf @@ -158,9 +158,15 @@ data "aws_iam_policy_document" "cluster_elb_sl_role_creation" { } } -resource "aws_iam_role_policy" "cluster_elb_sl_role_creation" { +resource "aws_iam_policy" "cluster_elb_sl_role_creation" { count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 name_prefix = "${var.cluster_name}-elb-sl-role-creation" - role = local.cluster_iam_role_name + description = "Permissions for EKS to create AWSServiceRoleForElasticLoadBalancing service-linked role" policy = data.aws_iam_policy_document.cluster_elb_sl_role_creation[0].json } + +resource "aws_iam_role_policy_attachment" "cluster_elb_sl_role_creation" { + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + policy_arn = aws_iam_policy.cluster_elb_sl_role_creation[0].arn + role = local.cluster_iam_role_name +} From d6f3a4265f870c20ec17a3c7f5e80a6514b1413b Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 6 Oct 2020 09:39:11 +0200 Subject: [PATCH 231/962] chore: update changelog for v13.0.0 release (#1034) --- CHANGELOG.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f74b1cfbd8..9bf7ae75c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,51 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v13.0.0] - 2020-10-05 +BUG FIXES: +- Use customer managed policy instead of inline policy for `cluster_elb_sl_role_creation` ([#1039](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1039)) +- More compatibility fixes for Terraform v0.13 and aws v3 ([#976](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/976)) +- Create `cluster_private_access` security group rules when it should ([#981](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/981)) +- random_pet with LT workers under 0.13.0 ([#940](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/940)) + +ENHANCEMENTS: +- Make the `cpu_credits` optional for workers launch template ([#1030](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1030)) +- update the `wait_for_cluster_cmd` logic to use `curl` if `wget` doesn't exist ([#1002](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1002)) + +FEATURES: +- Add `load_balancers` parameter to associate a CLB (Classic Load Balancer) to worker groups ASG ([#992](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/992)) +- Dynamic Partition for IRSA to support AWS-CN Deployments ([#1028](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1028)) +- Add AmazonEKSVPCResourceController to cluster policy to be able to set AWS Security Groups for pod ([#1011](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1011)) +- Cluster version is now a required variable. ([#972](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/972)) + +CI: +- Bump terraform pre-commit hook version and re-run terraform-docs with the latest version to fix the CI ([#1033](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1033)) +- fix CI lint job ([#973](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/973)) + +DOCS: +- Add important notes about the retry logic and the `wget` requirement ([#999](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/999)) +- Update README about `cluster_version` variable requirement ([#988](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/988)) +- Mixed spot + on-demand instance documentation ([#967](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/967)) +- Describe key_name is about AWS EC2 key pairs ([#970](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/970)) +- Better documentation of `cluster_id` output blocking ([#955](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/955)) + +BREAKING CHANGES: +- Default for `cluster_endpoint_private_access_cidrs` is now `null` instead of `["0.0.0.0/0"]`. It makes the variable required when `cluster_create_endpoint_private_access_sg_rule` is set to `true`. This will force everyone who want to have a private access to set explicitly their allowed subnets for the sake of the principle of least access by default. +- `cluster_version` variable is now required. + +NOTES: +- `credit_specification` for worker groups launch template can now be set to `null` so that we can use non burstable EC2 families +- Starting in v12.1.0 the `cluster_id` output depends on the +`wait_for_cluster` null resource. This means that initialisation of the +kubernetes provider will be blocked until the cluster is really ready, +if the module is set to manage the aws_auth ConfigMap and user followed +the typical Usage Example. kubernetes resources in the same plan do not +need to depend on anything explicitly. + + -## [v12.2.0] - 2020-07-12 +## [v12.2.0] - 2020-07-13 DOCS: - Update required IAM permissions list ([#936](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/936)) - Improve FAQ on how to deploy from Windows ([#927](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/927)) @@ -111,7 +154,8 @@ TESTS: - remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.2.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.0.0...HEAD +[v13.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.2.0...v13.0.0 [v12.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.1.0...v12.2.0 [v12.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.0.0...v12.1.0 [v12.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.1.0...v12.0.0 From 094e363362720c93009564813ea1bcfbe70e93a9 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 6 Oct 2020 14:26:50 +0200 Subject: [PATCH 232/962] fix: Use the amazon `ImageOwnerAlias` for worker ami owner instead of owner id (#1038) --- README.md | 4 ++-- variables.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index efd6681fbc..4ed76918a9 100644 --- a/README.md +++ b/README.md @@ -211,8 +211,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | | worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | -| worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"602401143452"` | no | -| worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"801119661308"` | no | +| worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"amazon"` | no | +| worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"amazon"` | no | | worker\_create\_cluster\_primary\_security\_group\_rules | Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group. | `bool` | `false` | no | | worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | `bool` | `false` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | `bool` | `true` | no | diff --git a/variables.tf b/variables.tf index f17bbd2b3a..dce05e123a 100644 --- a/variables.tf +++ b/variables.tf @@ -128,13 +128,13 @@ variable "worker_ami_name_filter_windows" { variable "worker_ami_owner_id" { description = "The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft')." type = string - default = "602401143452" // The ID of the owner of the official AWS EKS AMIs. + default = "amazon" } variable "worker_ami_owner_id_windows" { description = "The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft')." type = string - default = "801119661308" // The ID of the owner of the official AWS EKS Windows AMIs. + default = "amazon" } variable "worker_additional_security_group_ids" { From c54ce911dcb5ddec327e119a01e5b7a5667ee78a Mon Sep 17 00:00:00 2001 From: senthil Vivekanandan <60672460+senthilvv@users.noreply.github.com> Date: Mon, 12 Oct 2020 07:51:28 -0400 Subject: [PATCH 233/962] fix: Set IAM Path for `cluster_elb_sl_role_creation` IAM policy (#1045) --- cluster.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster.tf b/cluster.tf index 34b7314b5c..268de8013b 100644 --- a/cluster.tf +++ b/cluster.tf @@ -163,6 +163,7 @@ resource "aws_iam_policy" "cluster_elb_sl_role_creation" { name_prefix = "${var.cluster_name}-elb-sl-role-creation" description = "Permissions for EKS to create AWSServiceRoleForElasticLoadBalancing service-linked role" policy = data.aws_iam_policy_document.cluster_elb_sl_role_creation[0].json + path = var.iam_path } resource "aws_iam_role_policy_attachment" "cluster_elb_sl_role_creation" { From c75fbb0164427cbe92fe67c942140afba5936d2c Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 12 Oct 2020 21:50:22 +0200 Subject: [PATCH 234/962] fix: Fix cycle error during the destroy phase when we change workers order (#1043) NOTES: Keep in mind that changing the order of workers group is a destructive operation. All workers group are destroyed and recreated. If you want to do this safely, you should move then in state with `terraform state mv` until we manage workers groups as maps. --- workers.tf | 8 ++++++++ workers_launch_template.tf | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/workers.tf b/workers.tf index c6618a8451..a20b7d6486 100644 --- a/workers.tf +++ b/workers.tf @@ -295,6 +295,10 @@ resource "random_pet" "workers" { keepers = { lc_name = aws_launch_configuration.workers[count.index].name } + + lifecycle { + create_before_destroy = true + } } resource "aws_security_group" "workers" { @@ -409,6 +413,10 @@ resource "aws_iam_instance_profile" "workers" { ) path = var.iam_path + + lifecycle { + create_before_destroy = true + } } resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 2546393dc8..9d1ed28547 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -492,6 +492,7 @@ resource "random_pet" "workers_launch_template" { ) ) } + lifecycle { create_before_destroy = true } @@ -506,4 +507,8 @@ resource "aws_iam_instance_profile" "workers_launch_template" { local.default_iam_role_id, ) path = var.iam_path + + lifecycle { + create_before_destroy = true + } } From d97edde9cdad3a30628425868b87e8baca6c1d4f Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 12 Oct 2020 21:59:29 +0200 Subject: [PATCH 235/962] fix: Use splat syntax to avoid errors during destroy with an empty state (#1041) --- aws_auth.tf | 3 +-- cluster.tf | 2 +- local.tf | 6 +++--- workers.tf | 15 +++++++-------- workers_launch_template.tf | 14 +++++++------- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/aws_auth.tf b/aws_auth.tf index b583c069a9..a4d9211288 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -1,5 +1,4 @@ -data "aws_caller_identity" "current" { -} +data "aws_caller_identity" "current" {} locals { auth_launch_template_worker_roles = [ diff --git a/cluster.tf b/cluster.tf index 268de8013b..fd1efb01ff 100644 --- a/cluster.tf +++ b/cluster.tf @@ -64,7 +64,7 @@ resource "null_resource" "wait_for_cluster" { count = var.create_eks && var.manage_aws_auth ? 1 : 0 depends_on = [ - aws_eks_cluster.this[0], + aws_eks_cluster.this, aws_security_group_rule.cluster_private_access, ] diff --git a/local.tf b/local.tf index 393dd7a857..2554697dd0 100644 --- a/local.tf +++ b/local.tf @@ -146,10 +146,10 @@ locals { kubeconfig = var.create_eks ? templatefile("${path.module}/templates/kubeconfig.tpl", { kubeconfig_name = local.kubeconfig_name - endpoint = aws_eks_cluster.this[0].endpoint - cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data + endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] + cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] aws_authenticator_command = var.kubeconfig_aws_authenticator_command - aws_authenticator_command_args = length(var.kubeconfig_aws_authenticator_command_args) > 0 ? var.kubeconfig_aws_authenticator_command_args : ["token", "-i", aws_eks_cluster.this[0].name] + aws_authenticator_command_args = length(var.kubeconfig_aws_authenticator_command_args) > 0 ? var.kubeconfig_aws_authenticator_command_args : ["token", "-i", coalescelist(aws_eks_cluster.this[*].name, [""])[0]] aws_authenticator_additional_args = var.kubeconfig_aws_authenticator_additional_args aws_authenticator_env_variables = var.kubeconfig_aws_authenticator_env_variables }) : "" diff --git a/workers.tf b/workers.tf index a20b7d6486..5c05981415 100644 --- a/workers.tf +++ b/workers.tf @@ -6,7 +6,7 @@ resource "aws_autoscaling_group" "workers" { "-", compact( [ - aws_eks_cluster.this[0].name, + coalescelist(aws_eks_cluster.this[*].name, [""])[0], lookup(var.worker_groups[count.index], "name", count.index), lookup(var.worker_groups[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers[count.index].id : "" ] @@ -112,16 +112,16 @@ resource "aws_autoscaling_group" "workers" { [ { "key" = "Name" - "value" = "${aws_eks_cluster.this[0].name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg" + "value" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg" "propagate_at_launch" = true }, { - "key" = "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" + "key" = "kubernetes.io/cluster/${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}" "value" = "owned" "propagate_at_launch" = true }, { - "key" = "k8s.io/cluster/${aws_eks_cluster.this[0].name}" + "key" = "k8s.io/cluster/${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}" "value" = "owned" "propagate_at_launch" = true }, @@ -148,7 +148,7 @@ resource "aws_autoscaling_group" "workers" { resource "aws_launch_configuration" "workers" { count = var.create_eks ? local.worker_group_count : 0 - name_prefix = "${aws_eks_cluster.this[0].name}-${lookup(var.worker_groups[count.index], "name", count.index)}" + name_prefix = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup(var.worker_groups[count.index], "name", count.index)}" associate_public_ip_address = lookup( var.worker_groups[count.index], "public_ip", @@ -262,7 +262,6 @@ resource "aws_launch_configuration" "workers" { ) delete_on_termination = lookup(ebs_block_device.value, "delete_on_termination", true) } - } lifecycle { @@ -394,7 +393,7 @@ resource "aws_security_group_rule" "cluster_primary_ingress_workers" { resource "aws_iam_role" "workers" { count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - name_prefix = var.workers_role_name != "" ? null : aws_eks_cluster.this[0].name + name_prefix = var.workers_role_name != "" ? null : coalescelist(aws_eks_cluster.this[*].name, [""])[0] name = var.workers_role_name != "" ? var.workers_role_name : null assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json permissions_boundary = var.permissions_boundary @@ -405,7 +404,7 @@ resource "aws_iam_role" "workers" { resource "aws_iam_instance_profile" "workers" { count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_count : 0 - name_prefix = aws_eks_cluster.this[0].name + name_prefix = coalescelist(aws_eks_cluster.this[*].name, [""])[0] role = lookup( var.worker_groups[count.index], "iam_role_id", diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 9d1ed28547..89301246c0 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -6,7 +6,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { "-", compact( [ - aws_eks_cluster.this[0].name, + coalescelist(aws_eks_cluster.this[*].name, [""])[0], lookup(var.worker_groups_launch_template[count.index], "name", count.index), lookup(var.worker_groups_launch_template[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers_launch_template[count.index].id : "" ] @@ -189,7 +189,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { [ { "key" = "Name" - "value" = "${aws_eks_cluster.this[0].name}-${lookup( + "value" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -197,7 +197,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { "propagate_at_launch" = true }, { - "key" = "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" + "key" = "kubernetes.io/cluster/${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}" "value" = "owned" "propagate_at_launch" = true }, @@ -224,7 +224,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { resource "aws_launch_template" "workers_launch_template" { count = var.create_eks ? (local.worker_group_launch_template_count) : 0 - name_prefix = "${aws_eks_cluster.this[0].name}-${lookup( + name_prefix = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -427,7 +427,7 @@ resource "aws_launch_template" "workers_launch_template" { tags = merge( { - "Name" = "${aws_eks_cluster.this[0].name}-${lookup( + "Name" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -442,7 +442,7 @@ resource "aws_launch_template" "workers_launch_template" { tags = merge( { - "Name" = "${aws_eks_cluster.this[0].name}-${lookup( + "Name" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -500,7 +500,7 @@ resource "random_pet" "workers_launch_template" { resource "aws_iam_instance_profile" "workers_launch_template" { count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_launch_template_count : 0 - name_prefix = aws_eks_cluster.this[0].name + name_prefix = coalescelist(aws_eks_cluster.this[*].name, [""])[0] role = lookup( var.worker_groups_launch_template[count.index], "iam_role_id", From c7f88997d1a7e97052fe400877be71fc1e688c3e Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Sun, 25 Oct 2020 15:07:22 +0100 Subject: [PATCH 236/962] ci: Fix GitHub Actions CI macOS build errors (#1065) Temporary fix for https://github.com/actions/virtual-environments/issues/1811 --- .github/workflows/lint.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index bfb0ebad63..1efb366fc0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -21,11 +21,16 @@ jobs: docs: name: Docs - runs-on: macOS-latest + runs-on: macos-latest steps: - uses: actions/checkout@master - name: Install Deps - run: brew update && brew install pre-commit terraform-docs terraform + run: | + # Temporary fix for https://github.com/actions/virtual-environments/issues/1811 + brew untap local/homebrew-openssl + brew untap local/homebrew-python2 + brew update + brew install pre-commit terraform-docs terraform - name: Check Docs run: pre-commit run --show-diff-on-failure --all-files terraform_docs From 751c422330edd02744ae09efa8b8c289b4f3a364 Mon Sep 17 00:00:00 2001 From: Guillaume GILL Date: Sun, 25 Oct 2020 15:14:50 +0100 Subject: [PATCH 237/962] feat: Add kubernetes standard labels to avoid manual mistakes on the managed `aws-auth` configmap (#989) --- README.md | 1 + aws_auth.tf | 9 +++++++++ variables.tf | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/README.md b/README.md index 4ed76918a9..dddacf4b8b 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | +| aws\_auth\_additional\_labels | Additionnal kubernetes labels applied on aws-auth ConfigMap | `map(string)` | `{}` | no | | cluster\_create\_endpoint\_private\_access\_sg\_rule | Whether to create security group rules for the access to the Amazon EKS private API server endpoint. | `bool` | `false` | no | | cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | | cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | diff --git a/aws_auth.tf b/aws_auth.tf index a4d9211288..aa4cf482ef 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -67,6 +67,15 @@ resource "kubernetes_config_map" "aws_auth" { metadata { name = "aws-auth" namespace = "kube-system" + labels = merge( + { + "app.kubernetes.io/managed-by" = "Terraform" + # / are replaced by . because label validator fails in this lib + # https://github.com/kubernetes/apimachinery/blob/1bdd76d09076d4dc0362456e59c8f551f5f24a72/pkg/util/validation/validation.go#L166 + "terraform.io/module" = "terraform-aws-modules.eks.aws" + }, + var.aws_auth_additional_labels + ) } data = { diff --git a/variables.tf b/variables.tf index dce05e123a..051d3b1d5a 100644 --- a/variables.tf +++ b/variables.tf @@ -47,6 +47,12 @@ variable "manage_aws_auth" { default = true } +variable "aws_auth_additional_labels" { + description = "Additionnal kubernetes labels applied on aws-auth ConfigMap" + default = {} + type = map(string) +} + variable "map_accounts" { description = "Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format." type = list(string) From ca7593a6dbf167c5e77e506e73797756441c2f6c Mon Sep 17 00:00:00 2001 From: Paul Pop Date: Wed, 28 Oct 2020 21:37:21 +0000 Subject: [PATCH 238/962] fix: The type of the output `cloudwatch_log_group_name` should be a string instead of a list of strings (#1061) NOTES: The output `cloudwatch_log_group_name` was incorrectly returning the log group name as a list of strings. As a workaround, people were using `module.eks_cluster.cloudwatch_log_group_name[0]` but that was totally inconsistent with output name. Those users can now use `module.eks_cluster.cloudwatch_log_group_name` directly. --- README.md | 2 +- outputs.tf | 2 +- variables.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dddacf4b8b..1ee3c0949a 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
| `[]` | no | | map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | | node\_groups | Map of map of node groups to create. See `node_groups` module's documentation for more details | `any` | `{}` | no | -| node\_groups\_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentaton for more details | `any` | `{}` | no | +| node\_groups\_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentation for more details | `any` | `{}` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | diff --git a/outputs.tf b/outputs.tf index a1ac5e579b..9d3e6953b0 100644 --- a/outputs.tf +++ b/outputs.tf @@ -58,7 +58,7 @@ output "cluster_primary_security_group_id" { output "cloudwatch_log_group_name" { description = "Name of cloudwatch log group created" - value = aws_cloudwatch_log_group.this[*].name + value = element(concat(aws_cloudwatch_log_group.this[*].name, list("")), 0) } output "kubeconfig" { diff --git a/variables.tf b/variables.tf index 051d3b1d5a..4c74731b20 100644 --- a/variables.tf +++ b/variables.tf @@ -318,7 +318,7 @@ variable "create_eks" { } variable "node_groups_defaults" { - description = "Map of values to be applied to all node groups. See `node_groups` module's documentaton for more details" + description = "Map of values to be applied to all node groups. See `node_groups` module's documentation for more details" type = any default = {} } From 58f753e9f8cdc7c2f0573e7385eda3ea5be79bb3 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 29 Oct 2020 01:13:43 +0100 Subject: [PATCH 239/962] ci: Use ubuntu-latest instead of MacOS for docs checks (#1074) --- .github/workflows/lint.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1efb366fc0..13f9c894af 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -21,18 +21,22 @@ jobs: docs: name: Docs - runs-on: macos-latest + runs-on: ubuntu-latest steps: - uses: actions/checkout@master - - name: Install Deps + - uses: actions/setup-python@v2 + - name: Install terraform-docs run: | - # Temporary fix for https://github.com/actions/virtual-environments/issues/1811 - brew untap local/homebrew-openssl - brew untap local/homebrew-python2 - brew update - brew install pre-commit terraform-docs terraform + # Get the download url of the latest version of terraform-docs + tf_docs_download_url=$(curl -s https://api.github.com/repos/terraform-docs/terraform-docs/releases/latest | jq -rc '.assets[] | select( .name | contains("linux-amd64")).browser_download_url') + mkdir -p $GITHUB_WORKSPACE/bin + curl -Lo $GITHUB_WORKSPACE/bin/terraform-docs $tf_docs_download_url + chmod +x $GITHUB_WORKSPACE/bin/terraform-docs + echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH - name: Check Docs - run: pre-commit run --show-diff-on-failure --all-files terraform_docs + uses: pre-commit/action@v2.0.0 + with: + extra_args: --show-diff-on-failure --all-files terraform_docs validate: name: Validate From 62a8f46ba70eab0516b3078d377dc7bc492abb97 Mon Sep 17 00:00:00 2001 From: Ken Kaizu Date: Thu, 29 Oct 2020 09:17:17 +0900 Subject: [PATCH 240/962] feat: Add `cloudwatch_log_group_arn` to outputs (#1071) --- README.md | 1 + outputs.tf | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 1ee3c0949a..4136b6039d 100644 --- a/README.md +++ b/README.md @@ -230,6 +230,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | |------|-------------| +| cloudwatch\_log\_group\_arn | Arn of cloudwatch log group created | | cloudwatch\_log\_group\_name | Name of cloudwatch log group created | | cluster\_arn | The Amazon Resource Name (ARN) of the cluster. | | cluster\_certificate\_authority\_data | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | diff --git a/outputs.tf b/outputs.tf index 9d3e6953b0..8c71fbe61e 100644 --- a/outputs.tf +++ b/outputs.tf @@ -61,6 +61,11 @@ output "cloudwatch_log_group_name" { value = element(concat(aws_cloudwatch_log_group.this[*].name, list("")), 0) } +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = element(concat(aws_cloudwatch_log_group.this[*].arn, list("")), 0) +} + output "kubeconfig" { description = "kubectl config file contents for this EKS cluster." value = local.kubeconfig From 127a3a883189d887eac3d0c9e91b7aa335d53b77 Mon Sep 17 00:00:00 2001 From: philicious Date: Mon, 2 Nov 2020 08:19:10 +0100 Subject: [PATCH 241/962] feat: Add Launch Template support for Managed Node Groups (#997) NOTES: Managed Node Groups now support Launch Templates. The Launch Template it self is not managed by this module, so you have to create it by your self and pass it's id to this module. See docs and [`examples/launch_templates_with_managed_node_groups/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/launch_templates_with_managed_node_group) for more details. --- README.md | 4 +- .../disk_encryption_policy.tf | 77 +++++++++++++++ .../launchtemplate.tf | 89 ++++++++++++++++++ .../main.tf | 93 +++++++++++++++++++ .../templates/userdata.sh.tpl | 12 +++ .../variables.tf | 14 +++ local.tf | 1 + modules/node_groups/README.md | 2 + modules/node_groups/locals.tf | 16 ++-- modules/node_groups/node_groups.tf | 14 ++- modules/node_groups/random.tf | 1 + versions.tf | 2 +- 12 files changed, 314 insertions(+), 11 deletions(-) create mode 100644 examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf create mode 100644 examples/launch_templates_with_managed_node_groups/launchtemplate.tf create mode 100644 examples/launch_templates_with_managed_node_groups/main.tf create mode 100644 examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl create mode 100644 examples/launch_templates_with_managed_node_groups/variables.tf diff --git a/README.md b/README.md index 4136b6039d..44fbcfe752 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | terraform | >= 0.12.9, != 0.13.0 | -| aws | >= 2.55.0 | +| aws | >= 3.3.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | @@ -156,7 +156,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| aws | >= 2.55.0 | +| aws | >= 3.3.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | diff --git a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf new file mode 100644 index 0000000000..bfeb9e8a71 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf @@ -0,0 +1,77 @@ +// if you have used ASGs before, that role got auto-created already and you need to import to TF state +resource "aws_iam_service_linked_role" "autoscaling" { + aws_service_name = "autoscaling.amazonaws.com" + description = "Default Service-Linked Role enables access to AWS Services and Resources used or managed by Auto Scaling" +} + +data "aws_caller_identity" "current" {} + +// This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes +data "aws_iam_policy_document" "ebs_decryption" { + // copy of default KMS policy that lets you manage it + statement { + sid = "Enable IAM User Permissions" + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + } + + actions = [ + "kms:*" + ] + + resources = ["*"] + } + + // required for EKS + statement { + sid = "Allow service-linked role use of the CMK" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", // required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, // required for the cluster / persistentvolume-controller to create encrypted PVCs + ] + } + + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ] + + resources = ["*"] + } + + statement { + sid = "Allow attachment of persistent resources" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", // required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, // required for the cluster / persistentvolume-controller to create encrypted PVCs + ] + } + + actions = [ + "kms:CreateGrant" + ] + + resources = ["*"] + + condition { + test = "Bool" + variable = "kms:GrantIsForAWSResource" + values = ["true"] + } + + } +} diff --git a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf new file mode 100644 index 0000000000..390e91d6c1 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf @@ -0,0 +1,89 @@ +data "template_file" "launch_template_userdata" { + template = file("${path.module}/templates/userdata.sh.tpl") + + vars = { + cluster_name = local.cluster_name + endpoint = module.eks.cluster_endpoint + cluster_auth_base64 = module.eks.cluster_certificate_authority_data + + bootstrap_extra_args = "" + kubelet_extra_args = "" + } +} + +// this is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) +// there are several more options one could set but you probably dont need to modify them +// you can take the default and add your custom AMI and/or custom tags +// +// Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, +// then the default user-data for bootstrapping a cluster is merged in the copy. +resource "aws_launch_template" "default" { + name_prefix = "eks-example-" + description = "Default Launch-Template" + update_default_version = true + + block_device_mappings { + device_name = "/dev/xvda" + + ebs { + volume_size = 100 + volume_type = "gp2" + delete_on_termination = true + //encrypted = true + // enable this if you want to encrypt your node root volumes with a KMS/CMK. encryption of PVCs is handled via k8s StorageClass tho + // you also need to attach data.aws_iam_policy_document.ebs_decryption.json from the disk_encryption_policy.tf to the KMS/CMK key then !! + //kms_key_id = var.kms_key_arn + } + } + + instance_type = var.instance_type + + monitoring { + enabled = true + } + + network_interfaces { + associate_public_ip_address = false + delete_on_termination = true + security_groups = [module.eks.worker_security_group_id] + } + + //image_id = var.ami_id // if you want to use a custom AMI + + // if you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then + // you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 + // + // (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) + + // user_data = base64encode( + // data.template_file.launch_template_userdata.rendered, + // ) + + + // supplying custom tags to EKS instances is another use-case for LaunchTemplates + tag_specifications { + resource_type = "instance" + + tags = { + CustomTag = "EKS example" + } + } + + // supplying custom tags to EKS instances root volumes is another use-case for LaunchTemplates. (doesnt add tags to dynamically provisioned volumes via PVC tho) + tag_specifications { + resource_type = "volume" + + tags = { + CustomTag = "EKS example" + } + } + + // tag the LT itself + tags = { + CustomTag = "EKS example" + } + + lifecycle { + create_before_destroy = true + } +} diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf new file mode 100644 index 0000000000..a20b382ef6 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/main.tf @@ -0,0 +1,93 @@ +terraform { + required_version = ">= 0.12.9" +} + +provider "aws" { + version = ">= 3.3.0" + region = var.region +} + +provider "random" { + version = "~> 2.1" +} + +provider "local" { + version = "~>1.4" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false + version = "~> 1.11" +} + +data "aws_availability_zones" "available" { +} + +locals { + cluster_name = "test-eks-lt-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.47.0" + + name = "test-vpc" + cidr = "172.16.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"] + public_subnets = ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" // EKS adds this and TF would want to remove then later + } +} + +module "eks" { + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.private_subnets + vpc_id = module.vpc.vpc_id + + node_groups = { + example = { + desired_capacity = 1 + max_capacity = 15 + min_capacity = 1 + + launch_template_id = aws_launch_template.default.id + launch_template_version = aws_launch_template.default.default_version + + additional_tags = { + CustomTag = "EKS example" + } + } + } +} diff --git a/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl b/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl new file mode 100644 index 0000000000..6cbad7921b --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl @@ -0,0 +1,12 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" +#!/bin/bash +set -xe + +# Bootstrap and join the cluster +/etc/eks/bootstrap.sh --b64-cluster-ca '${cluster_auth_base64}' --apiserver-endpoint '${endpoint}' ${bootstrap_extra_args} --kubelet-extra-args "${kubelet_extra_args}" '${cluster_name}' + +--//-- diff --git a/examples/launch_templates_with_managed_node_groups/variables.tf b/examples/launch_templates_with_managed_node_groups/variables.tf new file mode 100644 index 0000000000..2d986867b1 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/variables.tf @@ -0,0 +1,14 @@ +variable "region" { + default = "eu-central-1" +} + +variable "instance_type" { + default = "t3.small" // smallest recommended, where ~1.1Gb of 2Gb memory is available for the Kubernetes pods after ‘warming up’ Docker, Kubelet, and OS + type = string +} + +variable "kms_key_arn" { + default = "" + description = "KMS key ARN to use if you want to encrypt EKS node root volumes" + type = string +} diff --git a/local.tf b/local.tf index 2554697dd0..cd6ac7e254 100644 --- a/local.tf +++ b/local.tf @@ -80,6 +80,7 @@ locals { # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device + launch_template_id = "" # The id of the launch template used for managed node_groups launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group launch_template_placement_tenancy = "default" # The placement tenancy for instances launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index c905c749f2..9596222d89 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -26,6 +26,8 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | instance\_type | Workers' instance type | string | `var.workers_group_defaults[instance_type]` | | k8s\_labels | Kubernetes labels | map(string) | No labels applied | | key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` | +| launch_template_id | The id of a aws_launch_template to use | string | No LT used | +| launch\_template_version | The version of the LT to use | string | none | | max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` | | min\_capacity | Min number of workers | number | `var.workers_group_defaults[asg_min_size]` | | name | Name of the node group | string | Auto generated | diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 43cf672ca0..222412d450 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -2,13 +2,15 @@ locals { # Merge defaults and per-group values to make code cleaner node_groups_expanded = { for k, v in var.node_groups : k => merge( { - desired_capacity = var.workers_group_defaults["asg_desired_capacity"] - iam_role_arn = var.default_iam_role_arn - instance_type = var.workers_group_defaults["instance_type"] - key_name = var.workers_group_defaults["key_name"] - max_capacity = var.workers_group_defaults["asg_max_size"] - min_capacity = var.workers_group_defaults["asg_min_size"] - subnets = var.workers_group_defaults["subnets"] + desired_capacity = var.workers_group_defaults["asg_desired_capacity"] + iam_role_arn = var.default_iam_role_arn + instance_type = var.workers_group_defaults["instance_type"] + key_name = var.workers_group_defaults["key_name"] + launch_template_id = var.workers_group_defaults["launch_template_id"] + launch_template_version = var.workers_group_defaults["launch_template_version"] + max_capacity = var.workers_group_defaults["asg_max_size"] + min_capacity = var.workers_group_defaults["asg_min_size"] + subnets = var.workers_group_defaults["subnets"] }, var.node_groups_defaults, v, diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 77fa02e4bc..ba7e2656f5 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -15,7 +15,7 @@ resource "aws_eks_node_group" "workers" { ami_type = lookup(each.value, "ami_type", null) disk_size = lookup(each.value, "disk_size", null) - instance_types = [each.value["instance_type"]] + instance_types = each.value["launch_template_id"] != "" ? [] : [each.value["instance_type"]] release_version = lookup(each.value, "ami_release_version", null) dynamic "remote_access" { @@ -30,6 +30,18 @@ resource "aws_eks_node_group" "workers" { } } + dynamic "launch_template" { + for_each = each.value["launch_template_id"] != "" ? [{ + id = each.value["launch_template_id"] + version = each.value["launch_template_version"] + }] : [] + + content { + id = launch_template.value["id"] + version = launch_template.value["version"] + } + } + version = lookup(each.value, "version", null) labels = merge( diff --git a/modules/node_groups/random.tf b/modules/node_groups/random.tf index 16c0583e1c..aae2c6dbd2 100644 --- a/modules/node_groups/random.tf +++ b/modules/node_groups/random.tf @@ -17,6 +17,7 @@ resource "random_pet" "node_groups" { )) subnet_ids = join("|", each.value["subnets"]) node_group_name = join("-", [var.cluster_name, each.key]) + launch_template = lookup(each.value, "launch_template_id", null) } depends_on = [var.ng_depends_on] diff --git a/versions.tf b/versions.tf index 27c659f933..83458a43c3 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.12.9, != 0.13.0" required_providers { - aws = ">= 2.55.0" + aws = ">= 3.3.0" local = ">= 1.4" null = ">= 2.1" template = ">= 2.1" From 571e4e7f4bca0c30a6d714a4cbebb9aaaf69c88a Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 2 Nov 2020 08:35:12 +0100 Subject: [PATCH 242/962] chore: Use `#` for comments. It's the recommended way to start comment and it's more idiomatic (#1079) --- .../disk_encryption_policy.tf | 16 +++---- .../launchtemplate.tf | 44 ++++++++++--------- .../main.tf | 2 +- .../variables.tf | 3 +- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf index bfeb9e8a71..0f51fb1fde 100644 --- a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf +++ b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf @@ -1,4 +1,4 @@ -// if you have used ASGs before, that role got auto-created already and you need to import to TF state +# if you have used ASGs before, that role got auto-created already and you need to import to TF state resource "aws_iam_service_linked_role" "autoscaling" { aws_service_name = "autoscaling.amazonaws.com" description = "Default Service-Linked Role enables access to AWS Services and Resources used or managed by Auto Scaling" @@ -6,9 +6,9 @@ resource "aws_iam_service_linked_role" "autoscaling" { data "aws_caller_identity" "current" {} -// This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes +# This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes data "aws_iam_policy_document" "ebs_decryption" { - // copy of default KMS policy that lets you manage it + # Copy of default KMS policy that lets you manage it statement { sid = "Enable IAM User Permissions" effect = "Allow" @@ -25,7 +25,7 @@ data "aws_iam_policy_document" "ebs_decryption" { resources = ["*"] } - // required for EKS + # Required for EKS statement { sid = "Allow service-linked role use of the CMK" effect = "Allow" @@ -33,8 +33,8 @@ data "aws_iam_policy_document" "ebs_decryption" { principals { type = "AWS" identifiers = [ - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", // required for the ASG to manage encrypted volumes for nodes - module.eks.cluster_iam_role_arn, // required for the cluster / persistentvolume-controller to create encrypted PVCs + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs ] } @@ -56,8 +56,8 @@ data "aws_iam_policy_document" "ebs_decryption" { principals { type = "AWS" identifiers = [ - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", // required for the ASG to manage encrypted volumes for nodes - module.eks.cluster_iam_role_arn, // required for the cluster / persistentvolume-controller to create encrypted PVCs + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs ] } diff --git a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf index 390e91d6c1..e66bad1a9f 100644 --- a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf +++ b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf @@ -11,12 +11,12 @@ data "template_file" "launch_template_userdata" { } } -// this is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) -// there are several more options one could set but you probably dont need to modify them -// you can take the default and add your custom AMI and/or custom tags -// -// Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, -// then the default user-data for bootstrapping a cluster is merged in the copy. +# This is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) +# there are several more options one could set but you probably dont need to modify them +# you can take the default and add your custom AMI and/or custom tags +# +# Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, +# then the default user-data for bootstrapping a cluster is merged in the copy. resource "aws_launch_template" "default" { name_prefix = "eks-example-" description = "Default Launch-Template" @@ -29,10 +29,11 @@ resource "aws_launch_template" "default" { volume_size = 100 volume_type = "gp2" delete_on_termination = true - //encrypted = true - // enable this if you want to encrypt your node root volumes with a KMS/CMK. encryption of PVCs is handled via k8s StorageClass tho - // you also need to attach data.aws_iam_policy_document.ebs_decryption.json from the disk_encryption_policy.tf to the KMS/CMK key then !! - //kms_key_id = var.kms_key_arn + # encrypted = true + + # Enable this if you want to encrypt your node root volumes with a KMS/CMK. encryption of PVCs is handled via k8s StorageClass tho + # you also need to attach data.aws_iam_policy_document.ebs_decryption.json from the disk_encryption_policy.tf to the KMS/CMK key then !! + # kms_key_id = var.kms_key_arn } } @@ -48,19 +49,20 @@ resource "aws_launch_template" "default" { security_groups = [module.eks.worker_security_group_id] } - //image_id = var.ami_id // if you want to use a custom AMI + # if you want to use a custom AMI + # image_id = var.ami_id - // if you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then - // you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 - // - // (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) + # If you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then + # you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 + # + # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) - // user_data = base64encode( - // data.template_file.launch_template_userdata.rendered, - // ) + # user_data = base64encode( + # data.template_file.launch_template_userdata.rendered, + # ) - // supplying custom tags to EKS instances is another use-case for LaunchTemplates + # Supplying custom tags to EKS instances is another use-case for LaunchTemplates tag_specifications { resource_type = "instance" @@ -69,7 +71,7 @@ resource "aws_launch_template" "default" { } } - // supplying custom tags to EKS instances root volumes is another use-case for LaunchTemplates. (doesnt add tags to dynamically provisioned volumes via PVC tho) + # Supplying custom tags to EKS instances root volumes is another use-case for LaunchTemplates. (doesnt add tags to dynamically provisioned volumes via PVC tho) tag_specifications { resource_type = "volume" @@ -78,7 +80,7 @@ resource "aws_launch_template" "default" { } } - // tag the LT itself + # Tag the LT itself tags = { CustomTag = "EKS example" } diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf index a20b382ef6..004aa07d1a 100644 --- a/examples/launch_templates_with_managed_node_groups/main.tf +++ b/examples/launch_templates_with_managed_node_groups/main.tf @@ -65,7 +65,7 @@ module "vpc" { enable_dns_hostnames = true private_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" // EKS adds this and TF would want to remove then later + "kubernetes.io/cluster/${local.cluster_name}" = "shared" # EKS adds this and TF would want to remove then later } } diff --git a/examples/launch_templates_with_managed_node_groups/variables.tf b/examples/launch_templates_with_managed_node_groups/variables.tf index 2d986867b1..6dcb269f4c 100644 --- a/examples/launch_templates_with_managed_node_groups/variables.tf +++ b/examples/launch_templates_with_managed_node_groups/variables.tf @@ -3,7 +3,8 @@ variable "region" { } variable "instance_type" { - default = "t3.small" // smallest recommended, where ~1.1Gb of 2Gb memory is available for the Kubernetes pods after ‘warming up’ Docker, Kubelet, and OS + # Smallest recommended, where ~1.1Gb of 2Gb memory is available for the Kubernetes pods after ‘warming up’ Docker, Kubelet, and OS + default = "t3.small" type = string } From 3379e5982fb24fe584f37b67c848555457b3a978 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 2 Nov 2020 08:42:10 +0100 Subject: [PATCH 243/962] chore: update changelog (#1080) --- CHANGELOG.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bf7ae75c9..8358b0caa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,32 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v13.1.0] - 2020-11-02 +FEATURES: +- Add Launch Template support for Managed Node Groups ([#997](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/997)) +- Add `cloudwatch_log_group_arn` to outputs ([#1071](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1071)) +- Add kubernetes standard labels to avoid manual mistakes on the managed `aws-auth` configmap ([#989](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/989)) + +CI: +- Use ubuntu-latest instead of MacOS for docs checks ([#1074](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1074)) +- Fix GitHub Actions CI macOS build errors ([#1065](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1065)) + +BUG FIXES: +- The type of the output `cloudwatch_log_group_name` should be a string instead of a list of strings ([#1061](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1061)) +- Use splat syntax to avoid errors during destroy with an empty state ([#1041](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1041)) +- Fix cycle error during the destroy phase when we change workers order ([#1043](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1043)) +- Set IAM Path for `cluster_elb_sl_role_creation` IAM policy ([#1045](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1045)) +- Use the amazon `ImageOwnerAlias` for worker ami owner instead of owner id ([#1038](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1038)) + +NOTES: +- Managed Node Groups now support Launch Templates. The Launch Template it self is not managed by this module, so you have to create it by your self and pass it's id to this module. See docs and [`examples/launch_templates_with_managed_node_groups/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/launch_templates_with_managed_node_group) for more details. +- The output `cloudwatch_log_group_name` was incorrectly returning the log group name as a list of strings. As a workaround, people were using `module.eks_cluster.cloudwatch_log_group_name[0]` but that was totally inconsistent with output name. Those users can now use `module.eks_cluster.cloudwatch_log_group_name` directly. +- Keep in mind that changing the order of workers group is a destructive operation. All workers group are destroyed and recreated. If you want to do this safely, you should move then in state with `terraform state mv` until we manage workers groups as maps. + + -## [v13.0.0] - 2020-10-05 +## [v13.0.0] - 2020-10-06 BUG FIXES: - Use customer managed policy instead of inline policy for `cluster_elb_sl_role_creation` ([#1039](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1039)) - More compatibility fixes for Terraform v0.13 and aws v3 ([#976](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/976)) @@ -154,7 +178,8 @@ TESTS: - remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.0.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.1.0...HEAD +[v13.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.0.0...v13.1.0 [v13.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.2.0...v13.0.0 [v12.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.1.0...v12.2.0 [v12.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.0.0...v12.1.0 From e8a060f5ae2986cf88630d3279c7d800c24446ad Mon Sep 17 00:00:00 2001 From: Ary Kleinerman Date: Mon, 2 Nov 2020 18:11:58 -0300 Subject: [PATCH 244/962] docs: Update autoscaler installation in IRSA example (#1063) --- examples/irsa/README.md | 4 +++- examples/irsa/cluster-autoscaler-chart-values.yaml | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/irsa/README.md b/examples/irsa/README.md index 0af4d7b440..fd30ad3e3e 100644 --- a/examples/irsa/README.md +++ b/examples/irsa/README.md @@ -28,7 +28,9 @@ Replace `` with your AWS account ID in `cluster-autoscaler-chart-val Install the chart using the provided values file: ``` -helm install --name cluster-autoscaler --namespace kube-system stable/cluster-autoscaler --values=cluster-autoscaler-chart-values.yaml +$ helm repo add autoscaler https://kubernetes.github.io/autoscaler +$ helm repo update +$ helm install cluster-autoscaler --namespace kube-system autoscaler/cluster-autoscaler-chart --values=cluster-autoscaler-chart-values.yaml ``` ## Verify diff --git a/examples/irsa/cluster-autoscaler-chart-values.yaml b/examples/irsa/cluster-autoscaler-chart-values.yaml index 71b18c43a0..03f411f373 100644 --- a/examples/irsa/cluster-autoscaler-chart-values.yaml +++ b/examples/irsa/cluster-autoscaler-chart-values.yaml @@ -2,8 +2,9 @@ awsRegion: us-west-2 rbac: create: true - serviceAccountAnnotations: - eks.amazonaws.com/role-arn: "arn:aws:iam:::role/cluster-autoscaler" + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: "arn:aws:iam:::role/cluster-autoscaler" autoDiscovery: clusterName: test-eks-irsa From fc40053c0cf3097c243e61866e505b51abdf6bb4 Mon Sep 17 00:00:00 2001 From: Pranas Kiziela Date: Tue, 3 Nov 2020 18:02:13 +0200 Subject: [PATCH 245/962] docs: Explain node_groups and worker_groups difference in FAQ (#1081) --- docs/faq.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index 29cb2ee215..fe3903b6d7 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -189,3 +189,11 @@ Kubelet restricts the allowed list of labels in the `kubernetes.io` namespace th Older configurations used labels like `kubernetes.io/lifecycle=spot` and this is no longer allowed. Use `node.kubernetes.io/lifecycle=spot` instead. Reference the `--node-labels` argument for your version of Kubenetes for the allowed prefixes. [Documentation for 1.16](https://v1-16.docs.kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) + +## What is the difference between `node_groups` and `worker_groups`? + +`node_groups` are [AWS-managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) (configures "Node Groups" that you can find on the EKS dashboard). This system is supposed to ease some of the lifecycle around upgrading nodes. Although they do not do this automatically and you still need to manually trigger the updates. + +`worker_groups` are [self-managed nodes](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) (provisions a typical "Autoscaling group" on EC2). It gives you full control over nodes in the cluster like using custom AMI for the nodes. As AWS says, "with worker groups the customer controls the data plane & AWS controls the control plane". + +Both can be used together in the same cluster. From 3cb8fa3d87aa7c57ed5a0f24e95747dfd62a4715 Mon Sep 17 00:00:00 2001 From: Brannon Dorsey Date: Sat, 7 Nov 2020 08:28:52 -0500 Subject: [PATCH 246/962] docs: Fix IRSA example when deploying cluster-autoscaler from the latest kubernetes/autoscaler helm repo (#1090) --- docs/spot-instances.md | 2 +- examples/irsa/README.md | 12 ++++++------ examples/irsa/cluster-autoscaler-chart-values.yaml | 3 +++ examples/irsa/locals.tf | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/spot-instances.md b/docs/spot-instances.md index 442c228ebc..f140fe55ea 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -8,7 +8,7 @@ You need to install a daemonset to catch the 2 minute warning before termination helm install stable/k8s-spot-termination-handler --namespace kube-system ``` -In the following examples at least 1 worker group that uses on-demand instances is included. This worker group has an added node label that can be used in scheduling. This could be used to schedule any workload not suitable for spot instances but is important for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) as it might be end up unscheduled when spot instances are terminated. You can add this to the values of the [cluster-autoscaler helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler): +In the following examples at least 1 worker group that uses on-demand instances is included. This worker group has an added node label that can be used in scheduling. This could be used to schedule any workload not suitable for spot instances but is important for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) as it might be end up unscheduled when spot instances are terminated. You can add this to the values of the [cluster-autoscaler helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler-chart): ```yaml nodeSelector: diff --git a/examples/irsa/README.md b/examples/irsa/README.md index fd30ad3e3e..3cbe5e67e9 100644 --- a/examples/irsa/README.md +++ b/examples/irsa/README.md @@ -1,6 +1,6 @@ # IAM Roles for Service Accounts -This example shows how to create an IAM role to be used for a Kubernetes `ServiceAccount`. It will create a policy and role to be used by the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) using the [public Helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler). +This example shows how to create an IAM role to be used for a Kubernetes `ServiceAccount`. It will create a policy and role to be used by the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) using the [public Helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler-chart). The AWS documentation for IRSA is here: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html @@ -38,15 +38,15 @@ $ helm install cluster-autoscaler --namespace kube-system autoscaler/cluster-aut Ensure the cluster-autoscaler pod is running: ``` -$ kubectl --namespace=kube-system get pods -l "app.kubernetes.io/name=aws-cluster-autoscaler" -NAME READY STATUS RESTARTS AGE -cluster-autoscaler-aws-cluster-autoscaler-5545d4b97-9ztpm 1/1 Running 0 3m +$ kubectl --namespace=kube-system get pods -l "app.kubernetes.io/name=aws-cluster-autoscaler-chart" +NAME READY STATUS RESTARTS AGE +cluster-autoscaler-aws-cluster-autoscaler-chart-5545d4b97-9ztpm 1/1 Running 0 3m ``` Observe the `AWS_*` environment variables that were added to the pod automatically by EKS: ``` -kubectl --namespace=kube-system get pods -l "app.kubernetes.io/name=aws-cluster-autoscaler" -o yaml | grep -A3 AWS_ROLE_ARN +kubectl --namespace=kube-system get pods -l "app.kubernetes.io/name=aws-cluster-autoscaler-chart" -o yaml | grep -A3 AWS_ROLE_ARN - name: AWS_ROLE_ARN value: arn:aws:iam::xxxxxxxxx:role/cluster-autoscaler @@ -57,7 +57,7 @@ kubectl --namespace=kube-system get pods -l "app.kubernetes.io/name=aws-cluster- Verify it is working by checking the logs, you should see that it has discovered the autoscaling group successfully: ``` -kubectl --namespace=kube-system logs -l "app.kubernetes.io/name=aws-cluster-autoscaler" +kubectl --namespace=kube-system logs -l "app.kubernetes.io/name=aws-cluster-autoscaler-chart" I0128 14:59:00.901513 1 auto_scaling_groups.go:354] Regenerating instance to ASG map for ASGs: [test-eks-irsa-worker-group-12020012814125354700000000e] I0128 14:59:00.969875 1 auto_scaling_groups.go:138] Registering ASG test-eks-irsa-worker-group-12020012814125354700000000e diff --git a/examples/irsa/cluster-autoscaler-chart-values.yaml b/examples/irsa/cluster-autoscaler-chart-values.yaml index 03f411f373..ccde4365a5 100644 --- a/examples/irsa/cluster-autoscaler-chart-values.yaml +++ b/examples/irsa/cluster-autoscaler-chart-values.yaml @@ -3,7 +3,10 @@ awsRegion: us-west-2 rbac: create: true serviceAccount: + # This value should match local.k8s_service_account_name in locals.tf + name: cluster-autoscaler-aws-cluster-autoscaler-chart annotations: + # This value should match the ARN of the role created by module.iam_assumable_role_admin in irsa.tf eks.amazonaws.com/role-arn: "arn:aws:iam:::role/cluster-autoscaler" autoDiscovery: diff --git a/examples/irsa/locals.tf b/examples/irsa/locals.tf index 9cdc8af713..a0e5da0c28 100644 --- a/examples/irsa/locals.tf +++ b/examples/irsa/locals.tf @@ -1,5 +1,5 @@ locals { cluster_name = "test-eks-irsa" k8s_service_account_namespace = "kube-system" - k8s_service_account_name = "cluster-autoscaler-aws-cluster-autoscaler" + k8s_service_account_name = "cluster-autoscaler-aws-cluster-autoscaler-chart" } From bba7c151c81f83db9a948c853d5b27d39313982d Mon Sep 17 00:00:00 2001 From: huddy Date: Sat, 7 Nov 2020 21:20:22 +0000 Subject: [PATCH 247/962] feat: Tags passed into worker groups override tags from `var.tags` for Autoscaling Groups (#1092) NOTES: Tags that are passed into `var.worker_groups_launch_template` or `var.worker_groups` now override tags passed in via `var.tags` for Autoscaling Groups only. This allow ASG Tags to be overwritten, so that `propagate_at_launch` can be tweaked for a particular key. --- README.md | 2 +- local.tf | 9 --------- variables.tf | 2 +- workers.tf | 10 +++++++++- workers_launch_template.tf | 10 +++++++++- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 44fbcfe752..d4ae770a86 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | node\_groups\_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentation for more details | `any` | `{}` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | -| tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | +| tags | A map of tags to add to all resources. Tags added to launch coniguration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | | wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do if `command -v wget > /dev/null`; then wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; else curl -k -s $ENDPOINT/healthz >/dev/null && exit 0 || true;fi; sleep 5; done; echo TIMEOUT && exit 1"` | no | | wait\_for\_cluster\_interpreter | Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy. | `list(string)` |
[
"/bin/sh",
"-c"
]
| no | diff --git a/local.tf b/local.tf index cd6ac7e254..7e649a4f78 100644 --- a/local.tf +++ b/local.tf @@ -1,13 +1,4 @@ locals { - asg_tags = [ - for item in keys(var.tags) : - map( - "key", item, - "value", element(values(var.tags), index(keys(var.tags), item)), - "propagate_at_launch", "true" - ) - if item != "Name" - ] cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id cluster_primary_security_group_id = var.cluster_version >= 1.14 ? element(concat(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, list("")), 0) : null diff --git a/variables.tf b/variables.tf index 4c74731b20..965e127d9e 100644 --- a/variables.tf +++ b/variables.tf @@ -85,7 +85,7 @@ variable "subnets" { } variable "tags" { - description = "A map of tags to add to all resources." + description = "A map of tags to add to all resources. Tags added to launch coniguration or templates override these values for ASG Tags only." type = map(string) default = {} } diff --git a/workers.tf b/workers.tf index 5c05981415..b18ba70969 100644 --- a/workers.tf +++ b/workers.tf @@ -126,7 +126,15 @@ resource "aws_autoscaling_group" "workers" { "propagate_at_launch" = true }, ], - local.asg_tags, + [ + for tag_key, tag_value in var.tags : + map( + "key", tag_key, + "value", tag_value, + "propagate_at_launch", "true" + ) + if tag_key != "Name" && ! contains([for tag in lookup(var.worker_groups[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) + ], lookup( var.worker_groups[count.index], "tags", diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 89301246c0..0013fe3a8d 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -202,7 +202,15 @@ resource "aws_autoscaling_group" "workers_launch_template" { "propagate_at_launch" = true }, ], - local.asg_tags, + [ + for tag_key, tag_value in var.tags : + map( + "key", tag_key, + "value", tag_value, + "propagate_at_launch", "true" + ) + if tag_key != "Name" && ! contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) + ], lookup( var.worker_groups_launch_template[count.index], "tags", From 571da60aea10034cef20c435c2b49933ab5ede47 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Sat, 7 Nov 2020 22:29:24 +0100 Subject: [PATCH 248/962] fix: Change the default `launch_template_id` to `null` for Managed Node Groups (#1088) --- local.tf | 2 +- modules/node_groups/node_groups.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/local.tf b/local.tf index 7e649a4f78..bb23a6337f 100644 --- a/local.tf +++ b/local.tf @@ -71,7 +71,7 @@ locals { # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device - launch_template_id = "" # The id of the launch template used for managed node_groups + launch_template_id = null # The id of the launch template used for managed node_groups launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group launch_template_placement_tenancy = "default" # The placement tenancy for instances launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index ba7e2656f5..0cd2594287 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -15,7 +15,7 @@ resource "aws_eks_node_group" "workers" { ami_type = lookup(each.value, "ami_type", null) disk_size = lookup(each.value, "disk_size", null) - instance_types = each.value["launch_template_id"] != "" ? [] : [each.value["instance_type"]] + instance_types = each.value["launch_template_id"] != null ? [] : [each.value["instance_type"]] release_version = lookup(each.value, "ami_release_version", null) dynamic "remote_access" { @@ -31,7 +31,7 @@ resource "aws_eks_node_group" "workers" { } dynamic "launch_template" { - for_each = each.value["launch_template_id"] != "" ? [{ + for_each = each.value["launch_template_id"] != null ? [{ id = each.value["launch_template_id"] version = each.value["launch_template_version"] }] : [] From 0d77e30075e20ff16fa7357987a240e93f5f28ce Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Sat, 7 Nov 2020 23:03:12 +0100 Subject: [PATCH 249/962] feat: Add EKS Fargate support (#1067) Co-authored-by: Simon Gurcke Co-authored-by: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> --- README.md | 7 +++ aws_auth.tf | 6 +- examples/fargate/main.tf | 112 ++++++++++++++++++++++++++++++++++ examples/fargate/outputs.tf | 29 +++++++++ examples/fargate/variables.tf | 52 ++++++++++++++++ fargate.tf | 22 +++++++ modules/fargate/README.md | 54 ++++++++++++++++ modules/fargate/data.tf | 17 ++++++ modules/fargate/fargate.tf | 29 +++++++++ modules/fargate/locals.tf | 10 +++ modules/fargate/outputs.tf | 27 ++++++++ modules/fargate/variables.tf | 59 ++++++++++++++++++ outputs.tf | 20 ++++++ variables.tf | 18 ++++++ 14 files changed, 460 insertions(+), 2 deletions(-) create mode 100644 examples/fargate/main.tf create mode 100644 examples/fargate/outputs.tf create mode 100644 examples/fargate/variables.tf create mode 100644 fargate.tf create mode 100644 modules/fargate/README.md create mode 100644 modules/fargate/data.tf create mode 100644 modules/fargate/fargate.tf create mode 100644 modules/fargate/locals.tf create mode 100644 modules/fargate/outputs.tf create mode 100644 modules/fargate/variables.tf diff --git a/README.md b/README.md index d4ae770a86..1784ccab2f 100644 --- a/README.md +++ b/README.md @@ -187,8 +187,11 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | | create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | +| create\_fargate\_pod\_execution\_role | Controls if the EKS Fargate pod execution IAM role should be created. | `bool` | `true` | no | | eks\_oidc\_root\_ca\_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | | enable\_irsa | Whether to create OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no | +| fargate\_pod\_execution\_role\_name | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | +| fargate\_profiles | Fargate profiles to create. See `fargate_profile` keys section in fargate submodule's README.md for more details | `any` | `{}` | no | | iam\_path | If provided, all IAM roles will be created on this path. | `string` | `"/"` | no | | kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | `list(string)` | `[]` | no | | kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | `string` | `"aws-iam-authenticator"` | no | @@ -243,6 +246,10 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_security\_group\_id | Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console. | | cluster\_version | The Kubernetes server version for the EKS cluster. | | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | +| fargate\_iam\_role\_arn | IAM role ARN for EKS Fargate pods | +| fargate\_iam\_role\_name | IAM role name for EKS Fargate pods | +| fargate\_profile\_arns | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | +| fargate\_profile\_ids | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | | node\_groups | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys | diff --git a/aws_auth.tf b/aws_auth.tf index aa4cf482ef..91b0a23add 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -43,18 +43,20 @@ locals { local.auth_launch_template_worker_roles, local.auth_worker_roles, module.node_groups.aws_auth_roles, + module.fargate.aws_auth_roles, ) : { # Work around https://github.com/kubernetes-sigs/aws-iam-authenticator/issues/153 # Strip the leading slash off so that Terraform doesn't think it's a regex rolearn = replace(role["worker_role_arn"], replace(var.iam_path, "/^//", ""), "") - username = "system:node:{{EC2PrivateDNSName}}" + username = role["platform"] == "fargate" ? "system:node:{{SessionName}}" : "system:node:{{EC2PrivateDNSName}}" groups = tolist(concat( [ "system:bootstrappers", "system:nodes", ], - role["platform"] == "windows" ? ["eks:kube-proxy-windows"] : [] + role["platform"] == "windows" ? ["eks:kube-proxy-windows"] : [], + role["platform"] == "fargate" ? ["system:node-proxier"] : [], )) } ] diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf new file mode 100644 index 0000000000..f3f1aa0273 --- /dev/null +++ b/examples/fargate/main.tf @@ -0,0 +1,112 @@ +terraform { + required_version = ">= 0.12.6" +} + +provider "aws" { + version = ">= 2.28.1" + region = var.region +} + +provider "random" { + version = "~> 2.1" +} + +provider "local" { + version = "~> 1.2" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false + version = "~> 1.11" +} + +data "aws_availability_zones" "available" { +} + +locals { + cluster_name = "test-eks-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.47.0" + + name = "test-vpc" + cidr = "172.16.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"] + public_subnets = ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } +} + +module "eks" { + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.private_subnets + + tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + vpc_id = module.vpc.vpc_id + + fargate_profiles = { + example = { + namespace = "default" + + # Kubernetes labels for selection + # labels = { + # Environment = "test" + # GithubRepo = "terraform-aws-eks" + # GithubOrg = "terraform-aws-modules" + # } + + tags = { + Owner = "test" + } + } + } + + map_roles = var.map_roles + map_users = var.map_users + map_accounts = var.map_accounts +} diff --git a/examples/fargate/outputs.tf b/examples/fargate/outputs.tf new file mode 100644 index 0000000000..59aa57a2c9 --- /dev/null +++ b/examples/fargate/outputs.tf @@ -0,0 +1,29 @@ +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = module.eks.cluster_security_group_id +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = module.eks.kubeconfig +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = module.eks.config_map_aws_auth +} + +output "region" { + description = "AWS region." + value = var.region +} + +output "fargate_profile_arns" { + description = "Outputs from node groups" + value = module.eks.fargate_profile_arns +} diff --git a/examples/fargate/variables.tf b/examples/fargate/variables.tf new file mode 100644 index 0000000000..7085aeabd4 --- /dev/null +++ b/examples/fargate/variables.tf @@ -0,0 +1,52 @@ +variable "region" { + default = "us-west-2" +} + +variable "map_accounts" { + description = "Additional AWS account numbers to add to the aws-auth configmap." + type = list(string) + + default = [ + "777777777777", + "888888888888", + ] +} + +variable "map_roles" { + description = "Additional IAM roles to add to the aws-auth configmap." + type = list(object({ + rolearn = string + username = string + groups = list(string) + })) + + default = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] +} + +variable "map_users" { + description = "Additional IAM users to add to the aws-auth configmap." + type = list(object({ + userarn = string + username = string + groups = list(string) + })) + + default = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] +} diff --git a/fargate.tf b/fargate.tf new file mode 100644 index 0000000000..6ba03cbd89 --- /dev/null +++ b/fargate.tf @@ -0,0 +1,22 @@ +module "fargate" { + source = "./modules/fargate" + cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + create_eks = var.create_eks + create_fargate_pod_execution_role = var.create_fargate_pod_execution_role + fargate_pod_execution_role_name = var.fargate_pod_execution_role_name + fargate_profiles = var.fargate_profiles + iam_path = var.iam_path + iam_policy_arn_prefix = local.policy_arn_prefix + subnets = var.subnets + tags = var.tags + + # Hack to ensure ordering of resource creation. + # This is a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 + # Do not create node_groups before other resources are ready and removes race conditions + # Ensure these resources are created before "unlocking" the data source. + # Will be removed in Terraform 0.13 + eks_depends_on = [ + aws_eks_cluster.this, + kubernetes_config_map.aws_auth, + ] +} diff --git a/modules/fargate/README.md b/modules/fargate/README.md new file mode 100644 index 0000000000..f22dbb0731 --- /dev/null +++ b/modules/fargate/README.md @@ -0,0 +1,54 @@ +# eks `fargate` submodule + +Helper submodule to create and manage resources related to `aws_eks_fargate_profile`. + +## Assumptions +* Designed for use by the parent module and not directly by end users + +## `fargate_profile` keys +`fargate_profile` is a map of maps. Key of first level will be used as unique value for `for_each` resources and in the `aws_eks_fargate_profile` name. Inner map can take the below values. + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| name | Fargate profile name | `string` | Auto generated in the following format `[cluster_name]-fargate-[fargate_profile_map_key]`| no | +| namespace | Kubernetes namespace for selection | `string` | n/a | yes | +| labels | Key-value map of Kubernetes labels for selection | `map(string)` | `{}` | no | +| tags | Key-value map of resource tags. Will be merged with root module tags. | `map(string)` | `var.tags` | no | + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| aws | n/a | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_name | Name of the EKS cluster. | `string` | n/a | yes | +| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | +| create\_fargate\_pod\_execution\_role | Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created. | `bool` | `true` | no | +| eks\_depends\_on | List of references to other resources this submodule depends on. | `any` | `null` | no | +| fargate\_pod\_execution\_role\_name | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | +| fargate\_profiles | Fargate profiles to create. See `fargate_profile` keys section in README.md for more details | `any` | `{}` | no | +| iam\_path | IAM roles will be created on this path. | `string` | `"/"` | no | +| iam\_policy\_arn\_prefix | IAM policy prefix with the correct AWS partition. | `string` | n/a | yes | +| subnets | A list of subnets for the EKS Fargate profiles. | `list(string)` | `[]` | no | +| tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| aws\_auth\_roles | Roles for use in aws-auth ConfigMap | +| fargate\_profile\_arns | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | +| fargate\_profile\_ids | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | +| iam\_role\_arn | IAM role ARN for EKS Fargate pods | +| iam\_role\_name | IAM role name for EKS Fargate pods | + + diff --git a/modules/fargate/data.tf b/modules/fargate/data.tf new file mode 100644 index 0000000000..fc41f4924c --- /dev/null +++ b/modules/fargate/data.tf @@ -0,0 +1,17 @@ +data "aws_iam_policy_document" "eks_fargate_pod_assume_role" { + count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["eks-fargate-pods.amazonaws.com"] + } + } +} + +data "aws_iam_role" "custom_fargate_iam_role" { + count = local.create_eks && ! var.create_fargate_pod_execution_role ? 1 : 0 + name = var.fargate_pod_execution_role_name +} diff --git a/modules/fargate/fargate.tf b/modules/fargate/fargate.tf new file mode 100644 index 0000000000..f3592ba27f --- /dev/null +++ b/modules/fargate/fargate.tf @@ -0,0 +1,29 @@ +resource "aws_iam_role" "eks_fargate_pod" { + count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 + name_prefix = format("%s-fargate", var.cluster_name) + assume_role_policy = data.aws_iam_policy_document.eks_fargate_pod_assume_role[0].json + tags = var.tags + path = var.iam_path +} + +resource "aws_iam_role_policy_attachment" "eks_fargate_pod" { + count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 + policy_arn = "${var.iam_policy_arn_prefix}/AmazonEKSFargatePodExecutionRolePolicy" + role = aws_iam_role.eks_fargate_pod[0].name +} + +resource "aws_eks_fargate_profile" "this" { + for_each = local.create_eks ? local.fargate_profiles_expanded : {} + cluster_name = var.cluster_name + fargate_profile_name = lookup(each.value, "name", format("%s-fargate-%s", var.cluster_name, replace(each.key, "_", "-"))) + pod_execution_role_arn = local.pod_execution_role_arn + subnet_ids = var.subnets + tags = each.value.tags + + selector { + namespace = each.value.namespace + labels = lookup(each.value, "labels", null) + } + + depends_on = [var.eks_depends_on] +} diff --git a/modules/fargate/locals.tf b/modules/fargate/locals.tf new file mode 100644 index 0000000000..dbd72f0107 --- /dev/null +++ b/modules/fargate/locals.tf @@ -0,0 +1,10 @@ +locals { + create_eks = var.create_eks && length(var.fargate_profiles) > 0 + pod_execution_role_arn = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.arn, list("")), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.arn, list("")), 0) + pod_execution_role_name = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.name, list("")), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.name, list("")), 0) + + fargate_profiles_expanded = { for k, v in var.fargate_profiles : k => merge( + { tags = var.tags }, + v, + ) if var.create_eks } +} diff --git a/modules/fargate/outputs.tf b/modules/fargate/outputs.tf new file mode 100644 index 0000000000..b4a16319df --- /dev/null +++ b/modules/fargate/outputs.tf @@ -0,0 +1,27 @@ +output "fargate_profile_ids" { + description = "EKS Cluster name and EKS Fargate Profile names separated by a colon (:)." + value = [for f in aws_eks_fargate_profile.this : f.id] +} + +output "fargate_profile_arns" { + description = "Amazon Resource Name (ARN) of the EKS Fargate Profiles." + value = [for f in aws_eks_fargate_profile.this : f.arn] +} + +output "iam_role_name" { + description = "IAM role name for EKS Fargate pods" + value = local.pod_execution_role_name +} + +output "iam_role_arn" { + description = "IAM role ARN for EKS Fargate pods" + value = local.pod_execution_role_arn +} + +output "aws_auth_roles" { + description = "Roles for use in aws-auth ConfigMap" + value = [{ + worker_role_arn = local.pod_execution_role_arn + platform = "fargate" + }] +} diff --git a/modules/fargate/variables.tf b/modules/fargate/variables.tf new file mode 100644 index 0000000000..745f13c462 --- /dev/null +++ b/modules/fargate/variables.tf @@ -0,0 +1,59 @@ +variable "cluster_name" { + description = "Name of the EKS cluster." + type = string +} + +variable "create_eks" { + description = "Controls if EKS resources should be created (it affects almost all resources)" + type = bool + default = true +} + +variable "iam_path" { + description = "IAM roles will be created on this path." + type = string + default = "/" +} + +variable "iam_policy_arn_prefix" { + description = "IAM policy prefix with the correct AWS partition." + type = string +} + +variable "create_fargate_pod_execution_role" { + description = "Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created." + type = bool + default = true +} + +variable "fargate_pod_execution_role_name" { + description = "The IAM Role that provides permissions for the EKS Fargate Profile." + type = string + default = null +} + +variable "fargate_profiles" { + description = "Fargate profiles to create. See `fargate_profile` keys section in README.md for more details" + type = any + default = {} +} + +variable "subnets" { + description = "A list of subnets for the EKS Fargate profiles." + type = list(string) + default = [] +} + +variable "tags" { + description = "A map of tags to add to all resources." + type = map(string) + default = {} +} + +# Hack for a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 +# Will be removed in Terraform 0.13 with the support of module's `depends_on` https://github.com/hashicorp/terraform/issues/10462 +variable "eks_depends_on" { + description = "List of references to other resources this submodule depends on." + type = any + default = null +} diff --git a/outputs.tf b/outputs.tf index 8c71fbe61e..3cf145236d 100644 --- a/outputs.tf +++ b/outputs.tf @@ -166,6 +166,26 @@ output "worker_iam_role_arn" { )[0] } +output "fargate_profile_ids" { + description = "EKS Cluster name and EKS Fargate Profile names separated by a colon (:)." + value = module.fargate.fargate_profile_ids +} + +output "fargate_profile_arns" { + description = "Amazon Resource Name (ARN) of the EKS Fargate Profiles." + value = module.fargate.fargate_profile_arns +} + +output "fargate_iam_role_name" { + description = "IAM role name for EKS Fargate pods" + value = module.fargate.iam_role_name +} + +output "fargate_iam_role_arn" { + description = "IAM role ARN for EKS Fargate pods" + value = module.fargate.iam_role_arn +} + output "node_groups" { description = "Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys" value = module.node_groups.node_groups diff --git a/variables.tf b/variables.tf index 965e127d9e..e0675ae347 100644 --- a/variables.tf +++ b/variables.tf @@ -349,3 +349,21 @@ variable "cluster_encryption_config" { })) default = [] } + +variable "fargate_profiles" { + description = "Fargate profiles to create. See `fargate_profile` keys section in fargate submodule's README.md for more details" + type = any + default = {} +} + +variable "create_fargate_pod_execution_role" { + description = "Controls if the EKS Fargate pod execution IAM role should be created." + type = bool + default = true +} + +variable "fargate_pod_execution_role_name" { + description = "The IAM Role that provides permissions for the EKS Fargate Profile." + type = string + default = null +} From 669736f71bf11593d911d04d3f7cc0223b3b0df3 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Sat, 7 Nov 2020 23:10:37 +0100 Subject: [PATCH 250/962] chore: update changelog (#1093) --- CHANGELOG.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8358b0caa8..35e9405203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,24 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v13.2.0] - 2020-11-07 +FEATURES: +- Add EKS Fargate support ([#1067](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1067)) +- Tags passed into worker groups override tags from `var.tags` for Autoscaling Groups ([#1092](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1092)) + +BUG FIXES: +- Change the default `launch_template_id` to `null` for Managed Node Groups ([#1088](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1088)) + +DOCS: +- Fix IRSA example when deploying cluster-autoscaler from the latest kubernetes/autoscaler helm repo ([#1090](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1090)) +- Explain node_groups and worker_groups difference in FAQ ([#1081](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1081)) +- Update autoscaler installation in IRSA example ([#1063](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1063)) + +NOTES: +- Tags that are passed into `var.worker_groups_launch_template` or `var.worker_groups` now override tags passed in via `var.tags` for Autoscaling Groups only. This allow ASG Tags to be overwritten, so that `propagate_at_launch` can be tweaked for a particular key. + + ## [v13.1.0] - 2020-11-02 FEATURES: @@ -178,7 +196,8 @@ TESTS: - remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.1.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.0...HEAD +[v13.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.1.0...v13.2.0 [v13.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.0.0...v13.1.0 [v13.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.2.0...v13.0.0 [v12.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.1.0...v12.2.0 From 43101979ef139bc7e4d57844bb933f62dd528330 Mon Sep 17 00:00:00 2001 From: huddy Date: Thu, 12 Nov 2020 08:48:53 +0000 Subject: [PATCH 251/962] =?UTF-8?q?fix:=20Don=E2=80=99t=20add=20empty=20Ro?= =?UTF-8?q?les=20ARN=20in=20aws-auth=20configmap,=20specifically=20when=20?= =?UTF-8?q?no=20Fargate=20profiles=20are=20specified=20(#1096)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/fargate/outputs.tf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/fargate/outputs.tf b/modules/fargate/outputs.tf index b4a16319df..126ba6e385 100644 --- a/modules/fargate/outputs.tf +++ b/modules/fargate/outputs.tf @@ -20,8 +20,10 @@ output "iam_role_arn" { output "aws_auth_roles" { description = "Roles for use in aws-auth ConfigMap" - value = [{ - worker_role_arn = local.pod_execution_role_arn - platform = "fargate" - }] + value = [ + for i in range(1) : { + worker_role_arn = local.pod_execution_role_arn + platform = "fargate" + } if local.create_eks + ] } From 63217467a9adb2e10f6a66fa07072171b985c65c Mon Sep 17 00:00:00 2001 From: huddy Date: Thu, 12 Nov 2020 09:17:55 +0000 Subject: [PATCH 252/962] improvement: Tags passed into worker groups should also be excluded from Launch Template tag specification (#1095) --- workers_launch_template.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 0013fe3a8d..4e96279efe 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -456,7 +456,10 @@ resource "aws_launch_template" "workers_launch_template" { count.index, )}-eks_asg" }, - var.tags, + { for tag_key, tag_value in var.tags : + tag_key => tag_value + if tag_key != "Name" && ! contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) + } ) } From e5d26e1dcc41f859eb8d2be16460fd3b5b016412 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 12 Nov 2020 10:53:37 +0100 Subject: [PATCH 253/962] docs: Clarify usage of both AWS-Managed Node Groups and Self-Managed Worker Groups (#1094) --- docs/faq.md | 6 ++++++ examples/managed_node_groups/main.tf | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index fe3903b6d7..9fba65b7fa 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -197,3 +197,9 @@ Reference the `--node-labels` argument for your version of Kubenetes for the all `worker_groups` are [self-managed nodes](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) (provisions a typical "Autoscaling group" on EC2). It gives you full control over nodes in the cluster like using custom AMI for the nodes. As AWS says, "with worker groups the customer controls the data plane & AWS controls the control plane". Both can be used together in the same cluster. + +## I'm using both AWS-Managed node groups and Self-Managed worker groups and pods scheduled on a AWS Managed node groups are unable resolve DNS (even communication between pods) + +This happen because Core DNS can be scheduled on Self-Managed worker groups and by default, the terraform module doesn't create security group rules to ensure communication between pods schedulled on Self-Managed worker group and AWS-Managed node groups. + +You can set `var.worker_create_cluster_primary_security_group_rules` to `true` to create required rules. diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index 6d66aa21ea..eee25b9e01 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -112,6 +112,21 @@ module "eks" { } } + # Create security group rules to allow communication between pods on workers and pods in managed node groups. + # Set this to true if you have AWS-Managed node groups and Self-Managed worker groups. + # See https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1089 + + # worker_create_cluster_primary_security_group_rules = true + + # worker_groups_launch_template = [ + # { + # name = "worker-group-1" + # instance_type = "t2.small" + # asg_desired_capacity = 2 + # public_ip = true + # } + # ] + map_roles = var.map_roles map_users = var.map_users map_accounts = var.map_accounts From e4520d2c2e1b1a182d4abb8bdaf032534f5ed89f Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 12 Nov 2020 11:03:46 +0100 Subject: [PATCH 254/962] chore: update changelog (#1097) --- CHANGELOG.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35e9405203..e5b45b98be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,18 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v13.2.1] - 2020-11-12 +DOCS: +- Clarify usage of both AWS-Managed Node Groups and Self-Managed Worker Groups ([#1094](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1094)) + +ENHANCEMENTS: +- Tags passed into worker groups should also be excluded from Launch Template tag specification ([#1095](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1095)) + +BUG FIXES: +- Don’t add empty Roles ARN in aws-auth configmap, specifically when no Fargate profiles are specified ([#1096](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1096)) + + ## [v13.2.0] - 2020-11-07 FEATURES: @@ -196,7 +208,8 @@ TESTS: - remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.1...HEAD +[v13.2.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.0...v13.2.1 [v13.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.1.0...v13.2.0 [v13.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.0.0...v13.1.0 [v13.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v12.2.0...v13.0.0 From 942a88e18f1f8de34a21ff7f01889b420fcf9372 Mon Sep 17 00:00:00 2001 From: Eric Herot <400447+eherot@users.noreply.github.com> Date: Tue, 22 Dec 2020 04:27:38 -0500 Subject: [PATCH 255/962] docs: Fix a typo in the `aws-auth` section of the README (#1099) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1784ccab2f..a9165a5291 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ You also need to ensure your applications and add ons are updated, or workloads An example of harming update was the removal of several commonly used, but deprecated APIs, in Kubernetes 1.16. More information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). -By default, this module manage the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready, we implemented a retry logic with an `local-exec` provisioner and `wget` (by default) with failover to `curl`. +By default, this module manages the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready, we implemented a retry logic with an `local-exec` provisioner and `wget` (by default) with failover to `curl`. **If you want to manage your `aws-auth` configmap, ensure you have `wget` (or `curl`) and `/bin/sh` installed where you're running Terraform or set `wait_for_cluster_cmd` and `wait_for_cluster_interpreter` to match your needs.** From ea2f6982d5ed2d370191c2c4be386299d9bb5406 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 22 Dec 2020 11:32:13 +0100 Subject: [PATCH 256/962] chore: Fix terraform format for TF 0.14 (#1157) --- cluster.tf | 2 +- modules/fargate/data.tf | 2 +- workers.tf | 4 ++-- workers_launch_template.tf | 14 +++++++------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cluster.tf b/cluster.tf index fd1efb01ff..abe4e3ecf1 100644 --- a/cluster.tf +++ b/cluster.tf @@ -27,7 +27,7 @@ resource "aws_eks_cluster" "this" { delete = var.cluster_delete_timeout } - dynamic encryption_config { + dynamic "encryption_config" { for_each = toset(var.cluster_encryption_config) content { diff --git a/modules/fargate/data.tf b/modules/fargate/data.tf index fc41f4924c..ee8e0d8e96 100644 --- a/modules/fargate/data.tf +++ b/modules/fargate/data.tf @@ -12,6 +12,6 @@ data "aws_iam_policy_document" "eks_fargate_pod_assume_role" { } data "aws_iam_role" "custom_fargate_iam_role" { - count = local.create_eks && ! var.create_fargate_pod_execution_role ? 1 : 0 + count = local.create_eks && !var.create_fargate_pod_execution_role ? 1 : 0 name = var.fargate_pod_execution_role_name } diff --git a/workers.tf b/workers.tf index b18ba70969..e99d3aee78 100644 --- a/workers.tf +++ b/workers.tf @@ -133,7 +133,7 @@ resource "aws_autoscaling_group" "workers" { "value", tag_value, "propagate_at_launch", "true" ) - if tag_key != "Name" && ! contains([for tag in lookup(var.worker_groups[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) + if tag_key != "Name" && !contains([for tag in lookup(var.worker_groups[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) ], lookup( var.worker_groups[count.index], @@ -194,7 +194,7 @@ resource "aws_launch_configuration" "workers" { ebs_optimized = lookup( var.worker_groups[count.index], "ebs_optimized", - ! contains( + !contains( local.ebs_optimized_not_supported, lookup( var.worker_groups[count.index], diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 4e96279efe..165d9a98fc 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -93,7 +93,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { local.workers_group_defaults["health_check_grace_period"] ) - dynamic mixed_instances_policy { + dynamic "mixed_instances_policy" { iterator = item for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? list(var.worker_groups_launch_template[count.index]) : [] @@ -157,7 +157,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { } } - dynamic launch_template { + dynamic "launch_template" { iterator = item for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? [] : list(var.worker_groups_launch_template[count.index]) @@ -209,7 +209,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { "value", tag_value, "propagate_at_launch", "true" ) - if tag_key != "Name" && ! contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) + if tag_key != "Name" && !contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) ], lookup( var.worker_groups_launch_template[count.index], @@ -289,7 +289,7 @@ resource "aws_launch_template" "workers_launch_template" { ebs_optimized = lookup( var.worker_groups_launch_template[count.index], "ebs_optimized", - ! contains( + !contains( local.ebs_optimized_not_supported, lookup( var.worker_groups_launch_template[count.index], @@ -336,7 +336,7 @@ resource "aws_launch_template" "workers_launch_template" { ) } - dynamic placement { + dynamic "placement" { for_each = lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"]) != null ? [lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])] : [] content { @@ -349,7 +349,7 @@ resource "aws_launch_template" "workers_launch_template" { } } - dynamic instance_market_options { + dynamic "instance_market_options" { for_each = lookup(var.worker_groups_launch_template[count.index], "market_type", null) == null ? [] : list(lookup(var.worker_groups_launch_template[count.index], "market_type", null)) content { market_type = instance_market_options.value @@ -458,7 +458,7 @@ resource "aws_launch_template" "workers_launch_template" { }, { for tag_key, tag_value in var.tags : tag_key => tag_value - if tag_key != "Name" && ! contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) + if tag_key != "Name" && !contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) } ) } From b0f51fdbe5d75d87160b3341d1b81edb806f1ebf Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 22 Dec 2020 18:43:08 +0800 Subject: [PATCH 257/962] feat: Add permissions boundary to fargate execution IAM role (#1108) --- fargate.tf | 1 + modules/fargate/README.md | 1 + modules/fargate/fargate.tf | 11 ++++++----- modules/fargate/variables.tf | 6 ++++++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/fargate.tf b/fargate.tf index 6ba03cbd89..413c582a9c 100644 --- a/fargate.tf +++ b/fargate.tf @@ -5,6 +5,7 @@ module "fargate" { create_fargate_pod_execution_role = var.create_fargate_pod_execution_role fargate_pod_execution_role_name = var.fargate_pod_execution_role_name fargate_profiles = var.fargate_profiles + permissions_boundary = var.permissions_boundary iam_path = var.iam_path iam_policy_arn_prefix = local.policy_arn_prefix subnets = var.subnets diff --git a/modules/fargate/README.md b/modules/fargate/README.md index f22dbb0731..592ac2ee9e 100644 --- a/modules/fargate/README.md +++ b/modules/fargate/README.md @@ -38,6 +38,7 @@ No requirements. | fargate\_profiles | Fargate profiles to create. See `fargate_profile` keys section in README.md for more details | `any` | `{}` | no | | iam\_path | IAM roles will be created on this path. | `string` | `"/"` | no | | iam\_policy\_arn\_prefix | IAM policy prefix with the correct AWS partition. | `string` | n/a | yes | +| permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | | subnets | A list of subnets for the EKS Fargate profiles. | `list(string)` | `[]` | no | | tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | diff --git a/modules/fargate/fargate.tf b/modules/fargate/fargate.tf index f3592ba27f..f618e47cbd 100644 --- a/modules/fargate/fargate.tf +++ b/modules/fargate/fargate.tf @@ -1,9 +1,10 @@ resource "aws_iam_role" "eks_fargate_pod" { - count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 - name_prefix = format("%s-fargate", var.cluster_name) - assume_role_policy = data.aws_iam_policy_document.eks_fargate_pod_assume_role[0].json - tags = var.tags - path = var.iam_path + count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 + name_prefix = format("%s-fargate", var.cluster_name) + assume_role_policy = data.aws_iam_policy_document.eks_fargate_pod_assume_role[0].json + permissions_boundary = var.permissions_boundary + tags = var.tags + path = var.iam_path } resource "aws_iam_role_policy_attachment" "eks_fargate_pod" { diff --git a/modules/fargate/variables.tf b/modules/fargate/variables.tf index 745f13c462..acfd69bd96 100644 --- a/modules/fargate/variables.tf +++ b/modules/fargate/variables.tf @@ -38,6 +38,12 @@ variable "fargate_profiles" { default = {} } +variable "permissions_boundary" { + description = "If provided, all IAM roles will be created with this permissions boundary attached." + type = string + default = null +} + variable "subnets" { description = "A list of subnets for the EKS Fargate profiles." type = list(string) From ccacc1779284e36ebdbe7c366eaba17adf4280e6 Mon Sep 17 00:00:00 2001 From: svaranasi-traderev <45045920+svaranasi-traderev@users.noreply.github.com> Date: Tue, 22 Dec 2020 05:55:43 -0500 Subject: [PATCH 258/962] feat: Add customizable Auto Scaling Group health check type (#1118) --- local.tf | 1 + workers.tf | 5 +++++ workers_launch_template.tf | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/local.tf b/local.tf index bb23a6337f..210f8842dc 100644 --- a/local.tf +++ b/local.tf @@ -37,6 +37,7 @@ locals { asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. asg_recreate_on_change = false # Recreate the autoscaling group when the Launch Template or Launch Configuration change. default_cooldown = null # The amount of time, in seconds, after a scaling activity completes before another scaling activity can start. + health_check_type = null # Controls how health checking is done. Valid values are "EC2" or "ELB". health_check_grace_period = null # Time in seconds after instance comes into service before checking health. instance_type = "m4.large" # Size of the workers instances. spot_price = "" # Cost of spot instance. diff --git a/workers.tf b/workers.tf index e99d3aee78..8143e6a304 100644 --- a/workers.tf +++ b/workers.tf @@ -88,6 +88,11 @@ resource "aws_autoscaling_group" "workers" { "default_cooldown", local.workers_group_defaults["default_cooldown"] ) + health_check_type = lookup( + var.worker_groups[count.index], + "health_check_type", + local.workers_group_defaults["health_check_type"] + ) health_check_grace_period = lookup( var.worker_groups[count.index], "health_check_grace_period", diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 165d9a98fc..f0e0f1a252 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -87,6 +87,11 @@ resource "aws_autoscaling_group" "workers_launch_template" { "default_cooldown", local.workers_group_defaults["default_cooldown"] ) + health_check_type = lookup( + var.worker_groups_launch_template[count.index], + "health_check_type", + local.workers_group_defaults["health_check_type"] + ) health_check_grace_period = lookup( var.worker_groups_launch_template[count.index], "health_check_grace_period", From 7d6e672303a07ce2f806953f2d72d60907d4fd9c Mon Sep 17 00:00:00 2001 From: Siddarth Prakash <1428486+sidprak@users.noreply.github.com> Date: Tue, 22 Dec 2020 10:26:47 -0500 Subject: [PATCH 259/962] feat: Add support for throughput parameter for `gp3` volumes (#1146) --- README.md | 4 ++-- local.tf | 1 + versions.tf | 2 +- workers_launch_template.tf | 10 ++++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a9165a5291..0497dad756 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | terraform | >= 0.12.9, != 0.13.0 | -| aws | >= 3.3.0 | +| aws | >= 3.21.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | @@ -156,7 +156,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| aws | >= 3.3.0 | +| aws | >= 3.21.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | diff --git a/local.tf b/local.tf index 210f8842dc..a07aec85fd 100644 --- a/local.tf +++ b/local.tf @@ -45,6 +45,7 @@ locals { root_volume_size = "100" # root volume size of workers instances. root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp2', or 'io1' root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". + root_volume_throughput = null # The amount of throughput to provision for a gp3 volume. key_name = "" # The key pair name that should be used for the instances in the autoscaling group pre_userdata = "" # userdata to pre-append to the default userdata. userdata_template_file = "" # alternate template to use for userdata diff --git a/versions.tf b/versions.tf index 83458a43c3..2e92c9bed0 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.12.9, != 0.13.0" required_providers { - aws = ">= 3.3.0" + aws = ">= 3.21.0" local = ">= 1.4" null = ">= 2.1" template = ">= 2.1" diff --git a/workers_launch_template.tf b/workers_launch_template.tf index f0e0f1a252..d7c4b4156a 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -384,6 +384,11 @@ resource "aws_launch_template" "workers_launch_template" { "root_iops", local.workers_group_defaults["root_iops"], ) + throughput = lookup( + var.worker_groups_launch_template[count.index], + "root_volume_throughput", + local.workers_group_defaults["root_volume_throughput"], + ) encrypted = lookup( var.worker_groups_launch_template[count.index], "root_encrypted", @@ -419,6 +424,11 @@ resource "aws_launch_template" "workers_launch_template" { "iops", local.workers_group_defaults["root_iops"], ) + throughput = lookup( + block_device_mappings.value, + "throughput", + local.workers_group_defaults["root_volume_throughput"], + ) encrypted = lookup( block_device_mappings.value, "encrypted", From 5d90c8fa7a6faa2bde10eac24a8bd9330be4ae3a Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" <301855+masih@users.noreply.github.com> Date: Tue, 22 Dec 2020 15:50:04 +0000 Subject: [PATCH 260/962] docs: Fixes typo in variables description (#1154) --- README.md | 2 +- variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0497dad756..1034436d99 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | node\_groups\_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentation for more details | `any` | `{}` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | -| tags | A map of tags to add to all resources. Tags added to launch coniguration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | +| tags | A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | | vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | | wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do if `command -v wget > /dev/null`; then wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; else curl -k -s $ENDPOINT/healthz >/dev/null && exit 0 || true;fi; sleep 5; done; echo TIMEOUT && exit 1"` | no | | wait\_for\_cluster\_interpreter | Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy. | `list(string)` |
[
"/bin/sh",
"-c"
]
| no | diff --git a/variables.tf b/variables.tf index e0675ae347..bf173e4579 100644 --- a/variables.tf +++ b/variables.tf @@ -85,7 +85,7 @@ variable "subnets" { } variable "tags" { - description = "A map of tags to add to all resources. Tags added to launch coniguration or templates override these values for ASG Tags only." + description = "A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only." type = map(string) default = {} } From 618019e331387ffb129d32af4c736e75477c9094 Mon Sep 17 00:00:00 2001 From: nitro Date: Wed, 23 Dec 2020 05:13:01 -0500 Subject: [PATCH 261/962] docs: Make it more obvious that `var.cluster_iam_role_name` will allow reusing an existing IAM Role for the cluster. (#1133) --- README.md | 2 +- variables.tf | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1034436d99..f46ba5cec8 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_endpoint\_private\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS private API server endpoint. | `list(string)` | `null` | no | | cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | | cluster\_endpoint\_public\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | -| cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. | `string` | `""` | no | +| cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. Set this to reuse an existing IAM role. | `string` | `""` | no | | cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `""` | no | | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | diff --git a/variables.tf b/variables.tf index bf173e4579..9ff70f9a99 100644 --- a/variables.tf +++ b/variables.tf @@ -3,11 +3,13 @@ variable "cluster_enabled_log_types" { description = "A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html)" type = list(string) } + variable "cluster_log_kms_key_id" { default = "" description = "If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html)" type = string } + variable "cluster_log_retention_in_days" { default = 90 description = "Number of days to retain log events. Default retention - 90 days." @@ -288,7 +290,7 @@ variable "manage_cluster_iam_resources" { } variable "cluster_iam_role_name" { - description = "IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false." + description = "IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. Set this to reuse an existing IAM role." type = string default = "" } From 65777d0a37791dd1200d478e3bacc4c9a75f24c8 Mon Sep 17 00:00:00 2001 From: Lena-nfb <64463786+Lena-nfb@users.noreply.github.com> Date: Wed, 23 Dec 2020 12:08:19 +0000 Subject: [PATCH 262/962] fix: Merge tags from Fargate profiles with common tags from cluster (#1159) --- modules/fargate/fargate.tf | 1 - modules/fargate/locals.tf | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/fargate/fargate.tf b/modules/fargate/fargate.tf index f618e47cbd..d8e532709c 100644 --- a/modules/fargate/fargate.tf +++ b/modules/fargate/fargate.tf @@ -20,7 +20,6 @@ resource "aws_eks_fargate_profile" "this" { pod_execution_role_arn = local.pod_execution_role_arn subnet_ids = var.subnets tags = each.value.tags - selector { namespace = each.value.namespace labels = lookup(each.value, "labels", null) diff --git a/modules/fargate/locals.tf b/modules/fargate/locals.tf index dbd72f0107..4216953b77 100644 --- a/modules/fargate/locals.tf +++ b/modules/fargate/locals.tf @@ -4,7 +4,7 @@ locals { pod_execution_role_name = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.name, list("")), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.name, list("")), 0) fargate_profiles_expanded = { for k, v in var.fargate_profiles : k => merge( - { tags = var.tags }, v, + { tags = merge(var.tags, lookup(v, "tags", {})) }, ) if var.create_eks } } From 576aa293f5fbbe4acd9aa8ad09be2706a242a6bf Mon Sep 17 00:00:00 2001 From: Jonas Chevalier Date: Thu, 28 Jan 2021 21:46:53 +0000 Subject: [PATCH 263/962] feat: Allow the overwrite of subnets for Fargate profiles (#1117) NOTES: The EKS cluster can be provisioned with both private and public subnets. But Fargate only accepts private ones. This new variable allows to override the subnets to explicitly pass the private subnets to Fargate and work around that issue. --- examples/fargate/main.tf | 3 +++ modules/fargate/README.md | 1 + modules/fargate/fargate.tf | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf index f3f1aa0273..7fd0118c96 100644 --- a/examples/fargate/main.tf +++ b/examples/fargate/main.tf @@ -100,6 +100,9 @@ module "eks" { # GithubOrg = "terraform-aws-modules" # } + # using specific subnets instead of all the ones configured in eks + # subnets = ["subnet-0ca3e3d1234a56c78"] + tags = { Owner = "test" } diff --git a/modules/fargate/README.md b/modules/fargate/README.md index 592ac2ee9e..8b4c72f25e 100644 --- a/modules/fargate/README.md +++ b/modules/fargate/README.md @@ -14,6 +14,7 @@ Helper submodule to create and manage resources related to `aws_eks_fargate_prof | namespace | Kubernetes namespace for selection | `string` | n/a | yes | | labels | Key-value map of Kubernetes labels for selection | `map(string)` | `{}` | no | | tags | Key-value map of resource tags. Will be merged with root module tags. | `map(string)` | `var.tags` | no | +| subnets | List of subnet IDs. Will replace the root module subnets. | `list(string)` | `var.subnets` | no | ## Requirements diff --git a/modules/fargate/fargate.tf b/modules/fargate/fargate.tf index d8e532709c..43f616315a 100644 --- a/modules/fargate/fargate.tf +++ b/modules/fargate/fargate.tf @@ -18,7 +18,7 @@ resource "aws_eks_fargate_profile" "this" { cluster_name = var.cluster_name fargate_profile_name = lookup(each.value, "name", format("%s-fargate-%s", var.cluster_name, replace(each.key, "_", "-"))) pod_execution_role_arn = local.pod_execution_role_arn - subnet_ids = var.subnets + subnet_ids = lookup(each.value, "subnets", var.subnets) tags = each.value.tags selector { namespace = each.value.namespace From 76537d1b8d89dbe104ec32fb6ddd2d7c084b5e28 Mon Sep 17 00:00:00 2001 From: Jay Date: Thu, 28 Jan 2021 13:52:10 -0800 Subject: [PATCH 264/962] feat: Use `gp3` as default as it saves 20% and is more performant (#1134) BREAKING CHANGES: The default root volume type is now `gp3` as it saves 20% and is more performant --- local.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local.tf b/local.tf index a07aec85fd..0b9a6c72f4 100644 --- a/local.tf +++ b/local.tf @@ -43,7 +43,7 @@ locals { spot_price = "" # Cost of spot instance. placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". root_volume_size = "100" # root volume size of workers instances. - root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp2', or 'io1' + root_volume_type = "gp3" # root volume type of workers instances, can be "standard", "gp3", "gp2", or "io1" root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". root_volume_throughput = null # The amount of throughput to provision for a gp3 volume. key_name = "" # The key pair name that should be used for the instances in the autoscaling group From 8978997bb1d5a769f03eac764d4da3caddfb9c76 Mon Sep 17 00:00:00 2001 From: Jonathan Cole <58993913+jonathan-mothership@users.noreply.github.com> Date: Thu, 28 Jan 2021 14:08:29 -0800 Subject: [PATCH 265/962] feat: Add the SPOT support for Managed Node Groups (#1129) BREAKING CHANGES: To add add SPOT support for MNG, the `instance_type` is now a list and renamed as `instance_types`. This will probably rebuild existing Managed Node Groups. --- examples/managed_node_groups/main.tf | 3 ++- modules/node_groups/README.md | 3 ++- modules/node_groups/locals.tf | 2 +- modules/node_groups/node_groups.tf | 3 ++- modules/node_groups/random.tf | 5 ++++- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index eee25b9e01..9c13298aa4 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -100,7 +100,8 @@ module "eks" { max_capacity = 10 min_capacity = 1 - instance_type = "m5.large" + instance_types = ["m5.large"] + capacity_type = "SPOT" k8s_labels = { Environment = "test" GithubRepo = "terraform-aws-eks" diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 9596222d89..e7a1967dc6 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -20,10 +20,11 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | additional\_tags | Additional tags to apply to node group | map(string) | Only `var.tags` applied | | ami\_release\_version | AMI version of workers | string | Provider default behavior | | ami\_type | AMI Type. See Terraform or AWS docs | string | Provider default behavior | +| capacity\_type | Type of instance capacity to provision. Options are `ON_DEMAND` and `SPOT` | string | Provider default behavior | | desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` | | disk\_size | Workers' disk size | number | Provider default behavior | | iam\_role\_arn | IAM role ARN for workers | string | `var.default_iam_role_arn` | -| instance\_type | Workers' instance type | string | `var.workers_group_defaults[instance_type]` | +| instance\_types | Node group's instance type(s). Multiple types can be specified when `capacity_type="SPOT"`. | list | `[var.workers_group_defaults[instance_type]]` | | k8s\_labels | Kubernetes labels | map(string) | No labels applied | | key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` | | launch_template_id | The id of a aws_launch_template to use | string | No LT used | diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 222412d450..61c633d5db 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -4,7 +4,7 @@ locals { { desired_capacity = var.workers_group_defaults["asg_desired_capacity"] iam_role_arn = var.default_iam_role_arn - instance_type = var.workers_group_defaults["instance_type"] + instance_types = [var.workers_group_defaults["instance_type"]] key_name = var.workers_group_defaults["key_name"] launch_template_id = var.workers_group_defaults["launch_template_id"] launch_template_version = var.workers_group_defaults["launch_template_version"] diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 0cd2594287..68abb094d9 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -15,8 +15,9 @@ resource "aws_eks_node_group" "workers" { ami_type = lookup(each.value, "ami_type", null) disk_size = lookup(each.value, "disk_size", null) - instance_types = each.value["launch_template_id"] != null ? [] : [each.value["instance_type"]] + instance_types = lookup(each.value, "instance_types", null) release_version = lookup(each.value, "ami_release_version", null) + capacity_type = lookup(each.value, "capacity_type", null) dynamic "remote_access" { for_each = each.value["key_name"] != "" ? [{ diff --git a/modules/node_groups/random.tf b/modules/node_groups/random.tf index aae2c6dbd2..c8b36697f4 100644 --- a/modules/node_groups/random.tf +++ b/modules/node_groups/random.tf @@ -7,8 +7,11 @@ resource "random_pet" "node_groups" { keepers = { ami_type = lookup(each.value, "ami_type", null) disk_size = lookup(each.value, "disk_size", null) - instance_type = each.value["instance_type"] + capacity_type = lookup(each.value, "capacity_type", null) iam_role_arn = each.value["iam_role_arn"] + instance_types = join("|", compact( + lookup(each.value, "instance_types", []) + )) key_name = each.value["key_name"] From d48ff6dccb89ba8b324358412de54f4e4714ae4d Mon Sep 17 00:00:00 2001 From: Suresh Kumar Nalluru Date: Thu, 28 Jan 2021 14:14:53 -0800 Subject: [PATCH 266/962] feat: Add support for `service_ipv4_cidr` for the EKS cluster (#1139) --- README.md | 1 + cluster.tf | 4 ++++ variables.tf | 6 ++++++ 3 files changed, 11 insertions(+) diff --git a/README.md b/README.md index f46ba5cec8..60fd7d47f1 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | | cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | | cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | +| cluster\_service\_ipv4\_cidr | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no | | cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | | create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | diff --git a/cluster.tf b/cluster.tf index abe4e3ecf1..53a696bdee 100644 --- a/cluster.tf +++ b/cluster.tf @@ -22,6 +22,10 @@ resource "aws_eks_cluster" "this" { public_access_cidrs = var.cluster_endpoint_public_access_cidrs } + kubernetes_network_config { + service_ipv4_cidr = var.cluster_service_ipv4_cidr + } + timeouts { create = var.cluster_create_timeout delete = var.cluster_delete_timeout diff --git a/variables.tf b/variables.tf index 9ff70f9a99..56fa8e6b3c 100644 --- a/variables.tf +++ b/variables.tf @@ -369,3 +369,9 @@ variable "fargate_pod_execution_role_name" { type = string default = null } + +variable "cluster_service_ipv4_cidr" { + description = "service ipv4 cidr for the kubernetes cluster" + type = string + default = null +} From 8912ae9a8b6de778fa96adf72b52cc75a8e704b7 Mon Sep 17 00:00:00 2001 From: Peter Parada Date: Thu, 28 Jan 2021 23:22:53 +0100 Subject: [PATCH 267/962] docs: Fix typos in README and CONTRIBUTING (#1167) --- .github/CONTRIBUTING.md | 8 ++++---- README.md | 2 +- variables.tf | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5ab6a91687..65df33743e 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -8,18 +8,18 @@ Please note we have a code of conduct, please follow it in all your interactions ## Pull Request Process 1. Ensure any install or build dependencies are removed before the end of the layer when doing a build. -2. Update the README.md with details of changes to the interface, this includes new environment variables, exposed ports, useful file locations and container parameters. -3. Once all outstanding comments and checklist items have been addressed, your contribution will be merged! Merged PRs will be included in the next release. The terraform-aws-eks mainteners takes care of updating the CHANGELOG as they merge. +2. Update the README.md with details of changes to the interface, this includes new environment variables, exposed ports, useful file locations, and container parameters. +3. Once all outstanding comments and checklist items have been addressed, your contribution will be merged! Merged PRs will be included in the next release. The terraform-aws-eks maintainers take care of updating the CHANGELOG as they merge. ## Checklists for contributions -- [ ] Add [sementics prefix](#semantic-pull-requests) to your PR or Commits (at leats one of your commit groups) +- [ ] Add [semantics prefix](#semantic-pull-requests) to your PR or Commits (at least one of your commit groups) - [ ] CI tests are passing - [ ] README.md has been updated after any changes to variables and outputs. See https://github.com/terraform-aws-modules/terraform-aws-eks/#doc-generation ## Semantic Pull Requests -To generate changelog, Pull Requests or Commits must have sementic and must follow conventional specs below: +To generate changelog, Pull Requests or Commits must have semantic and must follow conventional specs below: - `feat:` for new features - `fix:` for bug fixes diff --git a/README.md b/README.md index 60fd7d47f1..055e58f9e3 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | -| aws\_auth\_additional\_labels | Additionnal kubernetes labels applied on aws-auth ConfigMap | `map(string)` | `{}` | no | +| aws\_auth\_additional\_labels | Additional kubernetes labels applied on aws-auth ConfigMap | `map(string)` | `{}` | no | | cluster\_create\_endpoint\_private\_access\_sg\_rule | Whether to create security group rules for the access to the Amazon EKS private API server endpoint. | `bool` | `false` | no | | cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | | cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | diff --git a/variables.tf b/variables.tf index 56fa8e6b3c..e49c57ca49 100644 --- a/variables.tf +++ b/variables.tf @@ -50,7 +50,7 @@ variable "manage_aws_auth" { } variable "aws_auth_additional_labels" { - description = "Additionnal kubernetes labels applied on aws-auth ConfigMap" + description = "Additional kubernetes labels applied on aws-auth ConfigMap" default = {} type = map(string) } From 357e647ed5cc866c95bd921543759fdb73118499 Mon Sep 17 00:00:00 2001 From: oscr Date: Thu, 28 Jan 2021 23:26:35 +0100 Subject: [PATCH 268/962] docs: Change instance type from `t2` to `t3` in examples (#1169) --- examples/basic/main.tf | 4 ++-- examples/irsa/main.tf | 2 +- examples/launch_templates/main.tf | 4 ++-- examples/managed_node_groups/main.tf | 2 +- examples/secrets_encryption/main.tf | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/basic/main.tf b/examples/basic/main.tf index e5158af922..5cfedac88d 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -139,14 +139,14 @@ module "eks" { worker_groups = [ { name = "worker-group-1" - instance_type = "t2.small" + instance_type = "t3.small" additional_userdata = "echo foo bar" asg_desired_capacity = 2 additional_security_group_ids = [aws_security_group.worker_group_mgmt_one.id] }, { name = "worker-group-2" - instance_type = "t2.medium" + instance_type = "t3.medium" additional_userdata = "echo foo bar" additional_security_group_ids = [aws_security_group.worker_group_mgmt_two.id] asg_desired_capacity = 1 diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index 49992af7a9..fff2317293 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -65,7 +65,7 @@ module "eks" { worker_groups = [ { name = "worker-group-1" - instance_type = "t2.medium" + instance_type = "t3.medium" asg_desired_capacity = 1 tags = [ { diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 40558d4d85..c17d5a9dfd 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -72,13 +72,13 @@ module "eks" { worker_groups_launch_template = [ { name = "worker-group-1" - instance_type = "t2.small" + instance_type = "t3.small" asg_desired_capacity = 2 public_ip = true }, { name = "worker-group-2" - instance_type = "t2.medium" + instance_type = "t3.medium" asg_desired_capacity = 1 public_ip = true }, diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index 9c13298aa4..1bf66a0375 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -122,7 +122,7 @@ module "eks" { # worker_groups_launch_template = [ # { # name = "worker-group-1" - # instance_type = "t2.small" + # instance_type = "t3.small" # asg_desired_capacity = 2 # public_ip = true # } diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index 7462980ea7..337219c4a7 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -103,7 +103,7 @@ module "eks" { worker_groups = [ { name = "worker-group-1" - instance_type = "t2.small" + instance_type = "t3.small" additional_userdata = "echo foo bar" asg_desired_capacity = 2 }, From 6c08a9246345f5e2c1439b2bca029da1398d1ae1 Mon Sep 17 00:00:00 2001 From: lukemarsden Date: Thu, 28 Jan 2021 22:37:45 +0000 Subject: [PATCH 269/962] docs: Bump IRSA example dependencies to versions which work with TF 0.14 (#1184) --- examples/irsa/irsa.tf | 2 +- examples/irsa/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/irsa/irsa.tf b/examples/irsa/irsa.tf index d219c3cfc1..1775e33ecb 100644 --- a/examples/irsa/irsa.tf +++ b/examples/irsa/irsa.tf @@ -1,6 +1,6 @@ module "iam_assumable_role_admin" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "2.14.0" + version = "3.6.0" create_role = true role_name = "cluster-autoscaler" provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index fff2317293..de0186da8e 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -41,7 +41,7 @@ data "aws_caller_identity" "current" {} module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.47.0" + version = "2.64.0" name = "test-vpc" cidr = "10.0.0.0/16" azs = data.aws_availability_zones.available.names From 4fc609bd22459fce7863872ac032d9e89d75009f Mon Sep 17 00:00:00 2001 From: "Sean P. Kane" Date: Thu, 28 Jan 2021 14:45:41 -0800 Subject: [PATCH 270/962] feat: Add nitro enclave support for EKS (#1185) --- README.md | 4 ++-- local.tf | 1 + versions.tf | 2 +- workers_launch_template.tf | 8 ++++++++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 055e58f9e3..a513473299 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | terraform | >= 0.12.9, != 0.13.0 | -| aws | >= 3.21.0 | +| aws | >= 3.22.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | @@ -156,7 +156,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| aws | >= 3.21.0 | +| aws | >= 3.22.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | diff --git a/local.tf b/local.tf index 0b9a6c72f4..09d62f0df2 100644 --- a/local.tf +++ b/local.tf @@ -54,6 +54,7 @@ locals { additional_userdata = "" # userdata to append to the default userdata. ebs_optimized = true # sets whether to use ebs optimization on supported types. enable_monitoring = true # Enables/disables detailed monitoring. + enclave_support = false # Enables/disables enclave support public_ip = false # Associate a public ip address with a worker kubelet_extra_args = "" # This string is passed directly to kubelet if set. Useful for adding labels or taints. subnets = var.subnets # A list of subnets to place the worker nodes in. i.e. ["subnet-123", "subnet-456", "subnet-789"] diff --git a/versions.tf b/versions.tf index 2e92c9bed0..a5eb7b6c67 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.12.9, != 0.13.0" required_providers { - aws = ">= 3.21.0" + aws = ">= 3.22.0" local = ">= 1.4" null = ">= 2.1" template = ">= 2.1" diff --git a/workers_launch_template.tf b/workers_launch_template.tf index d7c4b4156a..f574fcc510 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -272,6 +272,14 @@ resource "aws_launch_template" "workers_launch_template" { )[count.index] } + enclave_options { + enabled = lookup( + var.worker_groups_launch_template[count.index], + "enclave_support", + local.workers_group_defaults["enclave_support"], + ) + } + image_id = lookup( var.worker_groups_launch_template[count.index], "ami_id", From 84a37617428af5bc7b457e7635d5b598ebb77815 Mon Sep 17 00:00:00 2001 From: myoung34 Date: Thu, 28 Jan 2021 16:54:38 -0600 Subject: [PATCH 271/962] improvement: Dont set -x in userdata to avoid printing sensitive informations in logs (#1187) --- .../templates/userdata.sh.tpl | 4 ++-- templates/userdata.sh.tpl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl b/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl index 6cbad7921b..41eeb0ba03 100644 --- a/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl +++ b/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl @@ -3,8 +3,8 @@ Content-Type: multipart/mixed; boundary="//" --// Content-Type: text/x-shellscript; charset="us-ascii" -#!/bin/bash -set -xe +#!/bin/bash +set -e # Bootstrap and join the cluster /etc/eks/bootstrap.sh --b64-cluster-ca '${cluster_auth_base64}' --apiserver-endpoint '${endpoint}' ${bootstrap_extra_args} --kubelet-extra-args "${kubelet_extra_args}" '${cluster_name}' diff --git a/templates/userdata.sh.tpl b/templates/userdata.sh.tpl index 6194a863e5..cf314b8800 100644 --- a/templates/userdata.sh.tpl +++ b/templates/userdata.sh.tpl @@ -1,4 +1,4 @@ -#!/bin/bash -xe +#!/bin/bash -e # Allow user supplied pre userdata code ${pre_userdata} From a8c402bb0e02c730974d5fc802ac29fe2e34296b Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 29 Jan 2021 00:30:05 +0100 Subject: [PATCH 272/962] docs: Update changelog generation to use custom sort with git-chglog v0.10.0 (#1202) --- .chglog/CHANGELOG.tpl.md | 8 ++++---- .chglog/config.yml | 10 +++++++++- Makefile | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.chglog/CHANGELOG.tpl.md b/.chglog/CHANGELOG.tpl.md index 01fa64d17a..1af11b8438 100644 --- a/.chglog/CHANGELOG.tpl.md +++ b/.chglog/CHANGELOG.tpl.md @@ -13,14 +13,14 @@ project adheres to [Semantic Versioning](http://semver.org/). {{ .Title }}: {{ range .Commits -}} {{- if .Subject -}} -- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject | upperFirst }} {{ end -}} {{ end }} {{ end -}} {{ else }} {{ range .Unreleased.Commits -}} {{- if .Subject -}} -- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject | upperFirst}} {{ end -}} {{ end }} {{ end -}} @@ -43,14 +43,14 @@ project adheres to [Semantic Versioning](http://semver.org/). {{ .Title }}: {{ range .Commits -}} {{- if .Subject -}} -- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject | upperFirst }} {{ end -}} {{ end }} {{ end -}} {{ else }} {{ range .Commits -}} {{- if .Subject -}} -- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject | upperFirst }} {{ end -}} {{ end }} {{ end -}} diff --git a/.chglog/config.yml b/.chglog/config.yml index 99485033c3..06f2cfb375 100644 --- a/.chglog/config.yml +++ b/.chglog/config.yml @@ -18,7 +18,15 @@ options: commit_groups: group_by: Type - sort_by: Type + sort_by: Custom + title_order: + - feat + - improvement + - refactor + - fix + - docs + - test + - ci title_maps: feat: FEATURES fix: BUG FIXES diff --git a/Makefile b/Makefile index 92c13f71f7..5120cc143a 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,10 @@ TAG_QUERY=v11.0.0.. scope ?= "minor" changelog-unrelease: - git-chglog -o $(CHANGELOG_FILE) $(TAG_QUERY) + git-chglog --no-case -o $(CHANGELOG_FILE) $(TAG_QUERY) changelog: - git-chglog -o $(CHANGELOG_FILE) --next-tag `$(SEMTAG) final -s $(scope) -o -f` $(TAG_QUERY) + git-chglog --no-case -o $(CHANGELOG_FILE) --next-tag `$(SEMTAG) final -s $(scope) -o -f` $(TAG_QUERY) release: $(SEMTAG) final -s $(scope) From a26c9fd0c9c880d5b99c438ad620e91dda957e10 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 29 Jan 2021 00:36:49 +0100 Subject: [PATCH 273/962] chore: update changelog (#1203) --- CHANGELOG.md | 127 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 82 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5b45b98be..85504e833d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,17 +10,52 @@ project adheres to [Semantic Versioning](http://semver.org/). - -## [v13.2.1] - 2020-11-12 + +## [v14.0.0] - 2021-01-29 +FEATURES: +- Add nitro enclave support for EKS ([#1185](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1185)) +- Add support for `service_ipv4_cidr` for the EKS cluster ([#1139](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1139)) +- Add the SPOT support for Managed Node Groups ([#1129](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1129)) +- Use `gp3` as default as it saves 20% and is more performant ([#1134](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1134)) +- Allow the overwrite of subnets for Fargate profiles ([#1117](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1117)) +- Add support for throughput parameter for `gp3` volumes ([#1146](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1146)) +- Add customizable Auto Scaling Group health check type ([#1118](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1118)) +- Add permissions boundary to fargate execution IAM role ([#1108](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1108)) + +ENHANCEMENTS: +- Dont set -x in userdata to avoid printing sensitive informations in logs ([#1187](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1187)) + +BUG FIXES: +- Merge tags from Fargate profiles with common tags from cluster ([#1159](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1159)) + DOCS: -- Clarify usage of both AWS-Managed Node Groups and Self-Managed Worker Groups ([#1094](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1094)) +- Update changelog generation to use custom sort with git-chglog v0.10.0 ([#1202](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1202)) +- Bump IRSA example dependencies to versions which work with TF 0.14 ([#1184](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1184)) +- Change instance type from `t2` to `t3` in examples ([#1169](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1169)) +- Fix typos in README and CONTRIBUTING ([#1167](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1167)) +- Make it more obvious that `var.cluster_iam_role_name` will allow reusing an existing IAM Role for the cluster. ([#1133](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1133)) +- Fixes typo in variables description ([#1154](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1154)) +- Fix a typo in the `aws-auth` section of the README ([#1099](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1099)) + +BREAKING CHANGES: +- To add add SPOT support for MNG, the `instance_type` is now a list and renamed as `instance_types`. This will probably rebuild existing Managed Node Groups. +- The default root volume type is now `gp3` as it saves 20% and is more performant + +NOTES: +- The EKS cluster can be provisioned with both private and public subnets. But Fargate only accepts private ones. This new variable allows to override the subnets to explicitly pass the private subnets to Fargate and work around that issue. + + +## [v13.2.1] - 2020-11-12 ENHANCEMENTS: - Tags passed into worker groups should also be excluded from Launch Template tag specification ([#1095](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1095)) BUG FIXES: - Don’t add empty Roles ARN in aws-auth configmap, specifically when no Fargate profiles are specified ([#1096](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1096)) +DOCS: +- Clarify usage of both AWS-Managed Node Groups and Self-Managed Worker Groups ([#1094](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1094)) + ## [v13.2.0] - 2020-11-07 @@ -47,10 +82,6 @@ FEATURES: - Add `cloudwatch_log_group_arn` to outputs ([#1071](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1071)) - Add kubernetes standard labels to avoid manual mistakes on the managed `aws-auth` configmap ([#989](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/989)) -CI: -- Use ubuntu-latest instead of MacOS for docs checks ([#1074](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1074)) -- Fix GitHub Actions CI macOS build errors ([#1065](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1065)) - BUG FIXES: - The type of the output `cloudwatch_log_group_name` should be a string instead of a list of strings ([#1061](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1061)) - Use splat syntax to avoid errors during destroy with an empty state ([#1041](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1041)) @@ -58,6 +89,10 @@ BUG FIXES: - Set IAM Path for `cluster_elb_sl_role_creation` IAM policy ([#1045](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1045)) - Use the amazon `ImageOwnerAlias` for worker ami owner instead of owner id ([#1038](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1038)) +CI: +- Use ubuntu-latest instead of MacOS for docs checks ([#1074](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1074)) +- Fix GitHub Actions CI macOS build errors ([#1065](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1065)) + NOTES: - Managed Node Groups now support Launch Templates. The Launch Template it self is not managed by this module, so you have to create it by your self and pass it's id to this module. See docs and [`examples/launch_templates_with_managed_node_groups/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/launch_templates_with_managed_node_group) for more details. - The output `cloudwatch_log_group_name` was incorrectly returning the log group name as a list of strings. As a workaround, people were using `module.eks_cluster.cloudwatch_log_group_name[0]` but that was totally inconsistent with output name. Those users can now use `module.eks_cluster.cloudwatch_log_group_name` directly. @@ -66,25 +101,21 @@ NOTES: ## [v13.0.0] - 2020-10-06 -BUG FIXES: -- Use customer managed policy instead of inline policy for `cluster_elb_sl_role_creation` ([#1039](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1039)) -- More compatibility fixes for Terraform v0.13 and aws v3 ([#976](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/976)) -- Create `cluster_private_access` security group rules when it should ([#981](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/981)) -- random_pet with LT workers under 0.13.0 ([#940](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/940)) - -ENHANCEMENTS: -- Make the `cpu_credits` optional for workers launch template ([#1030](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1030)) -- update the `wait_for_cluster_cmd` logic to use `curl` if `wget` doesn't exist ([#1002](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1002)) - FEATURES: - Add `load_balancers` parameter to associate a CLB (Classic Load Balancer) to worker groups ASG ([#992](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/992)) - Dynamic Partition for IRSA to support AWS-CN Deployments ([#1028](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1028)) - Add AmazonEKSVPCResourceController to cluster policy to be able to set AWS Security Groups for pod ([#1011](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1011)) - Cluster version is now a required variable. ([#972](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/972)) -CI: -- Bump terraform pre-commit hook version and re-run terraform-docs with the latest version to fix the CI ([#1033](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1033)) -- fix CI lint job ([#973](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/973)) +ENHANCEMENTS: +- Make the `cpu_credits` optional for workers launch template ([#1030](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1030)) +- Update the `wait_for_cluster_cmd` logic to use `curl` if `wget` doesn't exist ([#1002](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1002)) + +BUG FIXES: +- Use customer managed policy instead of inline policy for `cluster_elb_sl_role_creation` ([#1039](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1039)) +- More compatibility fixes for Terraform v0.13 and aws v3 ([#976](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/976)) +- Create `cluster_private_access` security group rules when it should ([#981](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/981)) +- Random_pet with LT workers under 0.13.0 ([#940](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/940)) DOCS: - Add important notes about the retry logic and the `wget` requirement ([#999](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/999)) @@ -93,6 +124,10 @@ DOCS: - Describe key_name is about AWS EC2 key pairs ([#970](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/970)) - Better documentation of `cluster_id` output blocking ([#955](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/955)) +CI: +- Bump terraform pre-commit hook version and re-run terraform-docs with the latest version to fix the CI ([#1033](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1033)) +- Fix CI lint job ([#973](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/973)) + BREAKING CHANGES: - Default for `cluster_endpoint_private_access_cidrs` is now `null` instead of `["0.0.0.0/0"]`. It makes the variable required when `cluster_create_endpoint_private_access_sg_rule` is set to `true`. This will force everyone who want to have a private access to set explicitly their allowed subnets for the sake of the principle of least access by default. - `cluster_version` variable is now required. @@ -109,21 +144,21 @@ need to depend on anything explicitly. ## [v12.2.0] - 2020-07-13 -DOCS: -- Update required IAM permissions list ([#936](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/936)) -- Improve FAQ on how to deploy from Windows ([#927](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/927)) -- autoscaler X.Y version must match ([#928](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/928)) - FEATURES: - IMDSv2 metadata configuration in Launch Templates ([#938](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/938)) -- worker launch templates and configurations depend on security group rules and IAM policies ([#933](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/933)) +- Worker launch templates and configurations depend on security group rules and IAM policies ([#933](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/933)) - Add IAM permissions for ELB svc-linked role creation by EKS cluster ([#902](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/902)) - Add a homemade `depends_on` for MNG submodule to ensure ordering of resource creation ([#867](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/867)) BUG FIXES: - Strip user Name tag from asg_tags [#946](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/946)) - Get `on_demand_allocation_strategy` from `local.workers_group_defaults` when deciding to use `mixed_instances_policy` ([#908](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/908)) -- remove unnecessary conditional in private access security group ([#915](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/915)) +- Remove unnecessary conditional in private access security group ([#915](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/915)) + +DOCS: +- Update required IAM permissions list ([#936](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/936)) +- Improve FAQ on how to deploy from Windows ([#927](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/927)) +- Autoscaler X.Y version must match ([#928](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/928)) NOTES: - Addition of the IMDSv2 metadata configuration block to Launch Templates will cause a diff to be generated for existing Launch Templates on first Terraform apply. The defaults match existing behaviour. @@ -149,6 +184,13 @@ NOTES: ## [v12.0.0] - 2020-05-09 +FEATURES: +- Create kubeconfig with non-executable permissions ([#864](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/864)) +- Change EKS default version to 1.16 ([#857](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/857)) + +ENHANCEMENTS: +- Remove dependency on external template provider ([#854](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/854)) + BUG FIXES: - Fix Launch Templates error with aws 2.61.0 ([#875](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/875)) - Use splat syntax for cluster name to avoid `(known after apply)` in managed node groups ([#868](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/868)) @@ -157,27 +199,20 @@ DOCS: - Add notes for Kubernetes 1.16 ([#873](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/873)) - Remove useless template provider in examples ([#863](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/863)) -FEATURES: -- Create kubeconfig with non-executable permissions ([#864](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/864)) -- Change EKS default version to 1.16 ([#857](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/857)) - -ENHANCEMENTS: -- Remove dependency on external template provider ([#854](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/854)) - BREAKING CHANGES: - The default `cluster_version` is now 1.16. Kubernetes 1.16 includes a number of deprecated API removals, and you need to ensure your applications and add ons are updated, or workloads could fail after the upgrade is complete. For more information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). For action you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). Please set explicitly your `cluster_version` to an older EKS version until your workloads are ready for Kubernetes 1.16. ## [v11.1.0] - 2020-04-23 +FEATURES: +- Add support for EC2 principal in assume worker role policy for China ([#827](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/827)) + BUG FIXES: - Add `vpc_config.cluster_security_group` output as primary cluster security group id ([#828](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/828)) - Wrap `local.configmap_roles.groups` with tolist() to avoid panic ([#846](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/846)) - Prevent `coalescelist` null argument error when destroying worker_group_launch_templates ([#842](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/842)) -FEATURES: -- Add support for EC2 principal in assume worker role policy for China ([#827](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/827)) - ## [v11.0.0] - 2020-03-31 @@ -187,28 +222,30 @@ FEATURES: - Add interpreter option to `wait_for_cluster_cmd` ([#795](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/795)) ENHANCEMENTS: +- Require kubernetes provider >=1.11.1 ([#784](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/784)) - Use `aws_partition` to build IAM policy ARNs ([#820](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/820)) - Generate `aws-auth` configmap's roles from Object. No more string concat. ([#790](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/790)) - Add timeout to default wait_for_cluster_cmd ([#791](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/791)) -- automate changelog management ([#786](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/786)) +- Automate changelog management ([#786](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/786)) BUG FIXES: - Fix destroy failure when talking to EKS endpoint on private network ([#815](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/815)) -- add ip address when manage_aws_auth is true and public_access is false ([#745](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/745)) +- Add ip address when manage_aws_auth is true and public_access is false ([#745](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/745)) - Add node_group direct dependency on eks_cluster ([#796](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/796)) - Do not recreate cluster when no SG given ([#798](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/798)) - Create `false` and avoid waiting forever for a non-existent cluster to respond ([#789](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/789)) -- fix git-chglog template to format changelog `Type` nicely ([#803](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/803)) -- fix git-chglog configuration ([#802](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/802)) +- Fix git-chglog template to format changelog `Type` nicely ([#803](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/803)) +- Fix git-chglog configuration ([#802](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/802)) + +TESTS: +- Remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -TESTS: -- remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) - -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.1...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v14.0.0...HEAD +[v14.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.1...v14.0.0 [v13.2.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.0...v13.2.1 [v13.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.1.0...v13.2.0 [v13.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.0.0...v13.1.0 From 2a6f07a0e8f9e43b669cf0aa69b4cc116cb973b7 Mon Sep 17 00:00:00 2001 From: Tjeu Kayim <15987676+TjeuKayim@users.noreply.github.com> Date: Fri, 16 Apr 2021 09:56:37 +0200 Subject: [PATCH 274/962] fix: Updated code and version requirements to work with Terraform 0.15 (#1165) --- .github/workflows/lint.yml | 82 ----- .github/workflows/pre-commit.yml | 102 +++++++ .gitignore | 47 ++- .pre-commit-config.yaml | 2 +- README.md | 286 +++++++++++------- examples/basic/main.tf | 26 +- examples/basic/versions.tf | 12 + examples/create_false/main.tf | 7 +- examples/create_false/versions.tf | 8 + examples/fargate/main.tf | 22 +- examples/fargate/versions.tf | 12 + examples/irsa/main.tf | 20 +- examples/irsa/versions.tf | 12 + examples/launch_templates/main.tf | 26 +- examples/launch_templates/versions.tf | 12 + .../main.tf | 26 +- .../versions.tf | 12 + examples/managed_node_groups/main.tf | 26 +- examples/managed_node_groups/versions.tf | 12 + examples/secrets_encryption/main.tf | 26 +- examples/secrets_encryption/versions.tf | 12 + examples/spot_instances/main.tf | 26 +- examples/spot_instances/versions.tf | 12 + local.tf | 2 +- modules/fargate/README.md | 54 ++-- modules/fargate/locals.tf | 4 +- modules/fargate/versions.tf | 7 + modules/node_groups/README.md | 42 ++- modules/node_groups/versions.tf | 8 + outputs.tf | 14 +- versions.tf | 2 +- workers.tf | 10 +- workers_launch_template.tf | 2 +- 33 files changed, 530 insertions(+), 443 deletions(-) delete mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pre-commit.yml create mode 100644 examples/basic/versions.tf create mode 100644 examples/create_false/versions.tf create mode 100644 examples/fargate/versions.tf create mode 100644 examples/irsa/versions.tf create mode 100644 examples/launch_templates/versions.tf create mode 100644 examples/launch_templates_with_managed_node_groups/versions.tf create mode 100644 examples/managed_node_groups/versions.tf create mode 100644 examples/secrets_encryption/versions.tf create mode 100644 examples/spot_instances/versions.tf create mode 100644 modules/fargate/versions.tf create mode 100644 modules/node_groups/versions.tf diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 13f9c894af..0000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Lint -on: [push, pull_request] - -jobs: - tflint: - name: TFLint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: TFLint - uses: docker://wata727/tflint - - fmt: - name: Code Format - runs-on: ubuntu-latest - container: - image: hashicorp/terraform:latest - steps: - - uses: actions/checkout@master - - run: terraform fmt --recursive -check=true - - docs: - name: Docs - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - uses: actions/setup-python@v2 - - name: Install terraform-docs - run: | - # Get the download url of the latest version of terraform-docs - tf_docs_download_url=$(curl -s https://api.github.com/repos/terraform-docs/terraform-docs/releases/latest | jq -rc '.assets[] | select( .name | contains("linux-amd64")).browser_download_url') - mkdir -p $GITHUB_WORKSPACE/bin - curl -Lo $GITHUB_WORKSPACE/bin/terraform-docs $tf_docs_download_url - chmod +x $GITHUB_WORKSPACE/bin/terraform-docs - echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH - - name: Check Docs - uses: pre-commit/action@v2.0.0 - with: - extra_args: --show-diff-on-failure --all-files terraform_docs - - validate: - name: Validate - runs-on: ubuntu-latest - container: - image: hashicorp/terraform:0.12.29 - steps: - - uses: actions/checkout@master - - name: Validate Code - env: - AWS_REGION: 'us-east-1' - TF_WARN_OUTPUT_ERRORS: 1 - TF_VAR_vpc_id: 'vpc-123456' - TF_VAR_subnets: '["subnet-12345a"]' - TF_VAR_workers_ami_id: 'ami-123456' - TF_VAR_cluster_name: 'test_cluster' - run: | - terraform init - terraform validate - - name: Validate Examples - run: | - for example in $(find examples -maxdepth 1 -mindepth 1 -type d); do - cd $example - terraform init - terraform validate - cd - - done - - minimum: - name: Minimum version check - runs-on: ubuntu-latest - container: - image: hashicorp/terraform:0.12.9 - steps: - - uses: actions/checkout@master - - name: Validate Code - env: - AWS_REGION: 'us-east-1' - TF_WARN_OUTPUT_ERRORS: 1 - run: | - sed -i -e 's/>=/=/' -e 's/ \(\d\+\.\d\+\)"/ \1.0"/' versions.tf - terraform init - terraform validate -var "region=${AWS_REGION}" -var "vpc_id=vpc-123456" -var "subnets=[\"subnet-12345a\"]" -var "workers_ami_id=ami-123456" -var "cluster_ingress_cidrs=[]" -var "cluster_name=test_cluster" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000000..59cd0a8967 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,102 @@ +name: Pre-Commit + +on: + pull_request: + push: + branches: + - master + +jobs: + # Min Terraform version(s) + getDirectories: + name: Get root directories + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v2 + - name: Build matrix + id: matrix + run: | + DIRS=$(python -c "import json; import glob; print(json.dumps([x.replace('/versions.tf', '') for x in glob.glob('./**/versions.tf', recursive=True)]))") + echo "::set-output name=directories::$DIRS" + outputs: + directories: ${{ steps.matrix.outputs.directories }} + + preCommitMinVersions: + name: Min TF validate + needs: getDirectories + runs-on: ubuntu-latest + strategy: + matrix: + directory: ${{ fromJson(needs.getDirectories.outputs.directories) }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v2 + - name: Terraform min/max versions + id: minMax + uses: clowdhaus/terraform-min-max@v1.0.1 + with: + directory: ${{ matrix.directory }} + - name: Install Terraform v${{ steps.minMax.outputs.minVersion }} + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: ${{ steps.minMax.outputs.minVersion }} + - name: Install pre-commit dependencies + run: pip install pre-commit + - name: Execute pre-commit + # Run only validate pre-commit check on min version supported + if: ${{ matrix.directory != '.' }} + run: + pre-commit run terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/* + - name: Execute pre-commit + # Run only validate pre-commit check on min version supported + if: ${{ matrix.directory == '.' }} + run: + pre-commit run terraform_validate --color=always --show-diff-on-failure --files $(ls *.tf) + + + # Max Terraform version + getBaseVersion: + name: Module max TF version + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Terraform min/max versions + id: minMax + uses: clowdhaus/terraform-min-max@v1.0.1 + outputs: + minVersion: ${{ steps.minMax.outputs.minVersion }} + maxVersion: ${{ steps.minMax.outputs.maxVersion }} + + preCommitMaxVersion: + name: Max TF pre-commit + runs-on: ubuntu-latest + needs: getBaseVersion + strategy: + fail-fast: false + matrix: + version: + - ${{ needs.getBaseVersion.outputs.maxVersion }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v2 + - name: Install Terraform v${{ matrix.version }} + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: ${{ matrix.version }} + - name: Install pre-commit dependencies + run: | + pip install pre-commit + curl -L "$(curl -s https://api.github.com/repos/terraform-docs/terraform-docs/releases/latest | grep -o -E "https://.+?-v0.12\..+?-linux-amd64" | head -n1)" > terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ + curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/ + - name: Execute pre-commit + # Run all pre-commit checks on max version supported + if: ${{ matrix.version == needs.getBaseVersion.outputs.maxVersion }} + run: pre-commit run --color=always --show-diff-on-failure --all-files diff --git a/.gitignore b/.gitignore index db3aec7ca3..4759a205e3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,41 @@ -*.tfvars -*.tfstate* -.terraform/ -**/inspec.lock -*.gem -Gemfile.lock -terraform.tfstate.d/ eks-admin-cluster-role-binding.yaml eks-admin-service-account.yaml -.idea/ -*.iml config-map-aws-auth*.yaml kubeconfig_* -*.swp + +################################################################# +# Default .gitignore content for all terraform-aws-modules below +################################################################# + +.DS_Store + +# Local .terraform directories +**/.terraform/* + +# Terraform lockfile +.terraform.lock.hcl + +# .tfstate files +*.tfstate +*.tfstate.* +*.tfplan + +# Crash log files +crash.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Ignore CLI configuration files +.terraformrc +terraform.rc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4949b1cbb5..6ef8a79bb9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.43.0 + rev: v1.48.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index a513473299..4687b2e829 100644 --- a/README.md +++ b/README.md @@ -86,9 +86,9 @@ data "aws_eks_cluster_auth" "cluster" { # In case of not creating the cluster, this will be an incompletely configured, unused provider, which poses no problem. provider "kubernetes" { - host = element(concat(data.aws_eks_cluster.cluster[*].endpoint, list("")), 0) - cluster_ca_certificate = base64decode(element(concat(data.aws_eks_cluster.cluster[*].certificate_authority.0.data, list("")), 0)) - token = element(concat(data.aws_eks_cluster_auth.cluster[*].token, list("")), 0) + host = element(concat(data.aws_eks_cluster.cluster[*].endpoint, [""]), 0) + cluster_ca_certificate = base64decode(element(concat(data.aws_eks_cluster.cluster[*].certificate_authority.0.data, [""]), 0)) + token = element(concat(data.aws_eks_cluster_auth.cluster[*].token, [""]), 0) load_config_file = false version = "1.10" } @@ -144,129 +144,189 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| terraform | >= 0.12.9, != 0.13.0 | -| aws | >= 3.22.0 | -| kubernetes | >= 1.11.1 | -| local | >= 1.4 | -| null | >= 2.1 | -| random | >= 2.1 | -| template | >= 2.1 | +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | +| [local](#requirement\_local) | >= 1.4 | +| [null](#requirement\_null) | >= 2.1 | +| [random](#requirement\_random) | >= 2.1 | +| [template](#requirement\_template) | >= 2.1 | ## Providers | Name | Version | |------|---------| -| aws | >= 3.22.0 | -| kubernetes | >= 1.11.1 | -| local | >= 1.4 | -| null | >= 2.1 | -| random | >= 2.1 | -| template | >= 2.1 | +| [aws](#provider\_aws) | >= 3.22.0 | +| [kubernetes](#provider\_kubernetes) | >= 1.11.1 | +| [local](#provider\_local) | >= 1.4 | +| [null](#provider\_null) | >= 2.1 | +| [random](#provider\_random) | >= 2.1 | +| [template](#provider\_template) | >= 2.1 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [fargate](#module\_fargate) | ./modules/fargate | | +| [node\_groups](#module\_node\_groups) | ./modules/node_groups | | + +## Resources + +| Name | Type | +|------|------| +| [aws_autoscaling_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | +| [aws_autoscaling_group.workers_launch_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | +| [aws_cloudwatch_log_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | +| [aws_iam_instance_profile.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_instance_profile.workers_launch_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_openid_connect_provider.oidc_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource | +| [aws_iam_policy.cluster_elb_sl_role_creation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cluster_AmazonEKSVPCResourceControllerPolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cluster_elb_sl_role_creation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.workers_additional_policies](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_launch_configuration.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_configuration) | resource | +| [aws_launch_template.workers_launch_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_security_group.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.cluster_egress_internet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.cluster_https_worker_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.cluster_primary_ingress_workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.cluster_private_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.workers_egress_internet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.workers_ingress_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.workers_ingress_cluster_https](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.workers_ingress_cluster_kubelet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.workers_ingress_cluster_primary](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.workers_ingress_self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | +| [local_file.kubeconfig](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [null_resource.wait_for_cluster](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_pet.workers](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | +| [random_pet.workers_launch_template](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | +| [aws_ami.eks_worker](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_ami.eks_worker_windows](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_instance_profile.custom_worker_group_iam_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_instance_profile) | data source | +| [aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_instance_profile) | data source | +| [aws_iam_policy_document.cluster_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cluster_elb_sl_role_creation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.workers_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_role.custom_cluster_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [template_file.launch_template_userdata](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | +| [template_file.userdata](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| attach\_worker\_cni\_policy | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | -| aws\_auth\_additional\_labels | Additional kubernetes labels applied on aws-auth ConfigMap | `map(string)` | `{}` | no | -| cluster\_create\_endpoint\_private\_access\_sg\_rule | Whether to create security group rules for the access to the Amazon EKS private API server endpoint. | `bool` | `false` | no | -| cluster\_create\_security\_group | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | -| cluster\_create\_timeout | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | -| cluster\_delete\_timeout | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | -| cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | -| cluster\_encryption\_config | Configuration block with encryption configuration for the cluster. See examples/secrets\_encryption/main.tf for example format |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | -| cluster\_endpoint\_private\_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | -| cluster\_endpoint\_private\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS private API server endpoint. | `list(string)` | `null` | no | -| cluster\_endpoint\_public\_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | -| cluster\_endpoint\_public\_access\_cidrs | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | -| cluster\_iam\_role\_name | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. Set this to reuse an existing IAM role. | `string` | `""` | no | -| cluster\_log\_kms\_key\_id | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `""` | no | -| cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | -| cluster\_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | -| cluster\_security\_group\_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | -| cluster\_service\_ipv4\_cidr | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no | -| cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | -| config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | -| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | -| create\_fargate\_pod\_execution\_role | Controls if the EKS Fargate pod execution IAM role should be created. | `bool` | `true` | no | -| eks\_oidc\_root\_ca\_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | -| enable\_irsa | Whether to create OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no | -| fargate\_pod\_execution\_role\_name | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | -| fargate\_profiles | Fargate profiles to create. See `fargate_profile` keys section in fargate submodule's README.md for more details | `any` | `{}` | no | -| iam\_path | If provided, all IAM roles will be created on this path. | `string` | `"/"` | no | -| kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | `list(string)` | `[]` | no | -| kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | `string` | `"aws-iam-authenticator"` | no | -| kubeconfig\_aws\_authenticator\_command\_args | Default arguments passed to the authenticator command. Defaults to [token -i $cluster\_name]. | `list(string)` | `[]` | no | -| kubeconfig\_aws\_authenticator\_env\_variables | Environment variables that should be used when executing the authenticator. e.g. { AWS\_PROFILE = "eks"}. | `map(string)` | `{}` | no | -| kubeconfig\_name | Override the default name used for items kubeconfig. | `string` | `""` | no | -| manage\_aws\_auth | Whether to apply the aws-auth configmap file. | `bool` | `true` | no | -| manage\_cluster\_iam\_resources | Whether to let the module manage cluster IAM resources. If set to false, cluster\_iam\_role\_name must be specified. | `bool` | `true` | no | -| manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam\_instance\_profile\_name must be specified for workers. | `bool` | `true` | no | -| map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | `list(string)` | `[]` | no | -| map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
| `[]` | no | -| map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | -| node\_groups | Map of map of node groups to create. See `node_groups` module's documentation for more details | `any` | `{}` | no | -| node\_groups\_defaults | Map of values to be applied to all node groups. See `node_groups` module's documentation for more details | `any` | `{}` | no | -| permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | -| subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | -| tags | A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | -| vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | -| wait\_for\_cluster\_cmd | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do if `command -v wget > /dev/null`; then wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; else curl -k -s $ENDPOINT/healthz >/dev/null && exit 0 || true;fi; sleep 5; done; echo TIMEOUT && exit 1"` | no | -| wait\_for\_cluster\_interpreter | Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy. | `list(string)` |
[
"/bin/sh",
"-c"
]
| no | -| worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | -| worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | -| worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | -| worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"amazon"` | no | -| worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"amazon"` | no | -| worker\_create\_cluster\_primary\_security\_group\_rules | Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group. | `bool` | `false` | no | -| worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | `bool` | `false` | no | -| worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | `bool` | `true` | no | -| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | -| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | -| worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | -| worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | `number` | `1025` | no | -| workers\_additional\_policies | Additional policies to be added to workers | `list(string)` | `[]` | no | -| workers\_group\_defaults | Override default values for target groups. See workers\_group\_defaults\_defaults in local.tf for valid keys. | `any` | `{}` | no | -| workers\_role\_name | User defined workers role name. | `string` | `""` | no | -| write\_kubeconfig | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | `bool` | `true` | no | +| [attach\_worker\_cni\_policy](#input\_attach\_worker\_cni\_policy) | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | +| [aws\_auth\_additional\_labels](#input\_aws\_auth\_additional\_labels) | Additional kubernetes labels applied on aws-auth ConfigMap | `map(string)` | `{}` | no | +| [cluster\_create\_endpoint\_private\_access\_sg\_rule](#input\_cluster\_create\_endpoint\_private\_access\_sg\_rule) | Whether to create security group rules for the access to the Amazon EKS private API server endpoint. | `bool` | `false` | no | +| [cluster\_create\_security\_group](#input\_cluster\_create\_security\_group) | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | +| [cluster\_create\_timeout](#input\_cluster\_create\_timeout) | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | +| [cluster\_delete\_timeout](#input\_cluster\_delete\_timeout) | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | +| [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | +| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster. See examples/secrets\_encryption/main.tf for example format |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | +| [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | +| [cluster\_endpoint\_private\_access\_cidrs](#input\_cluster\_endpoint\_private\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS private API server endpoint. | `list(string)` | `null` | no | +| [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | +| [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | +| [cluster\_iam\_role\_name](#input\_cluster\_iam\_role\_name) | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. Set this to reuse an existing IAM role. | `string` | `""` | no | +| [cluster\_log\_kms\_key\_id](#input\_cluster\_log\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `""` | no | +| [cluster\_log\_retention\_in\_days](#input\_cluster\_log\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | +| [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | +| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | +| [config\_output\_path](#input\_config\_output\_path) | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | +| [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | +| [create\_fargate\_pod\_execution\_role](#input\_create\_fargate\_pod\_execution\_role) | Controls if the EKS Fargate pod execution IAM role should be created. | `bool` | `true` | no | +| [eks\_oidc\_root\_ca\_thumbprint](#input\_eks\_oidc\_root\_ca\_thumbprint) | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | +| [enable\_irsa](#input\_enable\_irsa) | Whether to create OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no | +| [fargate\_pod\_execution\_role\_name](#input\_fargate\_pod\_execution\_role\_name) | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | +| [fargate\_profiles](#input\_fargate\_profiles) | Fargate profiles to create. See `fargate_profile` keys section in fargate submodule's README.md for more details | `any` | `{}` | no | +| [iam\_path](#input\_iam\_path) | If provided, all IAM roles will be created on this path. | `string` | `"/"` | no | +| [kubeconfig\_aws\_authenticator\_additional\_args](#input\_kubeconfig\_aws\_authenticator\_additional\_args) | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | `list(string)` | `[]` | no | +| [kubeconfig\_aws\_authenticator\_command](#input\_kubeconfig\_aws\_authenticator\_command) | Command to use to fetch AWS EKS credentials. | `string` | `"aws-iam-authenticator"` | no | +| [kubeconfig\_aws\_authenticator\_command\_args](#input\_kubeconfig\_aws\_authenticator\_command\_args) | Default arguments passed to the authenticator command. Defaults to [token -i $cluster\_name]. | `list(string)` | `[]` | no | +| [kubeconfig\_aws\_authenticator\_env\_variables](#input\_kubeconfig\_aws\_authenticator\_env\_variables) | Environment variables that should be used when executing the authenticator. e.g. { AWS\_PROFILE = "eks"}. | `map(string)` | `{}` | no | +| [kubeconfig\_name](#input\_kubeconfig\_name) | Override the default name used for items kubeconfig. | `string` | `""` | no | +| [manage\_aws\_auth](#input\_manage\_aws\_auth) | Whether to apply the aws-auth configmap file. | `bool` | `true` | no | +| [manage\_cluster\_iam\_resources](#input\_manage\_cluster\_iam\_resources) | Whether to let the module manage cluster IAM resources. If set to false, cluster\_iam\_role\_name must be specified. | `bool` | `true` | no | +| [manage\_worker\_iam\_resources](#input\_manage\_worker\_iam\_resources) | Whether to let the module manage worker IAM resources. If set to false, iam\_instance\_profile\_name must be specified for workers. | `bool` | `true` | no | +| [map\_accounts](#input\_map\_accounts) | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | `list(string)` | `[]` | no | +| [map\_roles](#input\_map\_roles) | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
| `[]` | no | +| [map\_users](#input\_map\_users) | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | +| [node\_groups](#input\_node\_groups) | Map of map of node groups to create. See `node_groups` module's documentation for more details | `any` | `{}` | no | +| [node\_groups\_defaults](#input\_node\_groups\_defaults) | Map of values to be applied to all node groups. See `node_groups` module's documentation for more details | `any` | `{}` | no | +| [permissions\_boundary](#input\_permissions\_boundary) | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | +| [subnets](#input\_subnets) | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | +| [tags](#input\_tags) | A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | +| [vpc\_id](#input\_vpc\_id) | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | +| [wait\_for\_cluster\_cmd](#input\_wait\_for\_cluster\_cmd) | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do if `command -v wget > /dev/null`; then wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; else curl -k -s $ENDPOINT/healthz >/dev/null && exit 0 || true;fi; sleep 5; done; echo TIMEOUT && exit 1"` | no | +| [wait\_for\_cluster\_interpreter](#input\_wait\_for\_cluster\_interpreter) | Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy. | `list(string)` |
[
"/bin/sh",
"-c"
]
| no | +| [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | +| [worker\_ami\_name\_filter](#input\_worker\_ami\_name\_filter) | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | +| [worker\_ami\_name\_filter\_windows](#input\_worker\_ami\_name\_filter\_windows) | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | +| [worker\_ami\_owner\_id](#input\_worker\_ami\_owner\_id) | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"amazon"` | no | +| [worker\_ami\_owner\_id\_windows](#input\_worker\_ami\_owner\_id\_windows) | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"amazon"` | no | +| [worker\_create\_cluster\_primary\_security\_group\_rules](#input\_worker\_create\_cluster\_primary\_security\_group\_rules) | Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group. | `bool` | `false` | no | +| [worker\_create\_initial\_lifecycle\_hooks](#input\_worker\_create\_initial\_lifecycle\_hooks) | Whether to create initial lifecycle hooks provided in worker groups. | `bool` | `false` | no | +| [worker\_create\_security\_group](#input\_worker\_create\_security\_group) | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | `bool` | `true` | no | +| [worker\_groups](#input\_worker\_groups) | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | +| [worker\_groups\_launch\_template](#input\_worker\_groups\_launch\_template) | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | +| [worker\_security\_group\_id](#input\_worker\_security\_group\_id) | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | +| [worker\_sg\_ingress\_from\_port](#input\_worker\_sg\_ingress\_from\_port) | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | `number` | `1025` | no | +| [workers\_additional\_policies](#input\_workers\_additional\_policies) | Additional policies to be added to workers | `list(string)` | `[]` | no | +| [workers\_group\_defaults](#input\_workers\_group\_defaults) | Override default values for target groups. See workers\_group\_defaults\_defaults in local.tf for valid keys. | `any` | `{}` | no | +| [workers\_role\_name](#input\_workers\_role\_name) | User defined workers role name. | `string` | `""` | no | +| [write\_kubeconfig](#input\_write\_kubeconfig) | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | `bool` | `true` | no | ## Outputs | Name | Description | |------|-------------| -| cloudwatch\_log\_group\_arn | Arn of cloudwatch log group created | -| cloudwatch\_log\_group\_name | Name of cloudwatch log group created | -| cluster\_arn | The Amazon Resource Name (ARN) of the cluster. | -| cluster\_certificate\_authority\_data | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | -| cluster\_endpoint | The endpoint for your EKS Kubernetes API. | -| cluster\_iam\_role\_arn | IAM role ARN of the EKS cluster. | -| cluster\_iam\_role\_name | IAM role name of the EKS cluster. | -| cluster\_id | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | -| cluster\_oidc\_issuer\_url | The URL on the EKS cluster OIDC Issuer | -| cluster\_primary\_security\_group\_id | The cluster primary security group ID created by the EKS cluster on 1.14 or later. Referred to as 'Cluster security group' in the EKS console. | -| cluster\_security\_group\_id | Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console. | -| cluster\_version | The Kubernetes server version for the EKS cluster. | -| config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | -| fargate\_iam\_role\_arn | IAM role ARN for EKS Fargate pods | -| fargate\_iam\_role\_name | IAM role name for EKS Fargate pods | -| fargate\_profile\_arns | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | -| fargate\_profile\_ids | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | -| kubeconfig | kubectl config file contents for this EKS cluster. | -| kubeconfig\_filename | The filename of the generated kubectl config. | -| node\_groups | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys | -| oidc\_provider\_arn | The ARN of the OIDC Provider if `enable_irsa = true`. | -| security\_group\_rule\_cluster\_https\_worker\_ingress | Security group rule responsible for allowing pods to communicate with the EKS cluster API. | -| worker\_iam\_instance\_profile\_arns | default IAM instance profile ARN for EKS worker groups | -| worker\_iam\_instance\_profile\_names | default IAM instance profile name for EKS worker groups | -| worker\_iam\_role\_arn | default IAM role ARN for EKS worker groups | -| worker\_iam\_role\_name | default IAM role name for EKS worker groups | -| worker\_security\_group\_id | Security group ID attached to the EKS workers. | -| workers\_asg\_arns | IDs of the autoscaling groups containing workers. | -| workers\_asg\_names | Names of the autoscaling groups containing workers. | -| workers\_default\_ami\_id | ID of the default worker group AMI | -| workers\_launch\_template\_arns | ARNs of the worker launch templates. | -| workers\_launch\_template\_ids | IDs of the worker launch templates. | -| workers\_launch\_template\_latest\_versions | Latest versions of the worker launch templates. | -| workers\_user\_data | User data of worker groups | - +| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | +| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | +| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster. | +| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | +| [cluster\_endpoint](#output\_cluster\_endpoint) | The endpoint for your EKS Kubernetes API. | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster. | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster. | +| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster OIDC Issuer | +| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | The cluster primary security group ID created by the EKS cluster on 1.14 or later. Referred to as 'Cluster security group' in the EKS console. | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console. | +| [cluster\_version](#output\_cluster\_version) | The Kubernetes server version for the EKS cluster. | +| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | +| [fargate\_iam\_role\_arn](#output\_fargate\_iam\_role\_arn) | IAM role ARN for EKS Fargate pods | +| [fargate\_iam\_role\_name](#output\_fargate\_iam\_role\_name) | IAM role name for EKS Fargate pods | +| [fargate\_profile\_arns](#output\_fargate\_profile\_arns) | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | +| [fargate\_profile\_ids](#output\_fargate\_profile\_ids) | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | +| [kubeconfig](#output\_kubeconfig) | kubectl config file contents for this EKS cluster. | +| [kubeconfig\_filename](#output\_kubeconfig\_filename) | The filename of the generated kubectl config. | +| [node\_groups](#output\_node\_groups) | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys | +| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true`. | +| [security\_group\_rule\_cluster\_https\_worker\_ingress](#output\_security\_group\_rule\_cluster\_https\_worker\_ingress) | Security group rule responsible for allowing pods to communicate with the EKS cluster API. | +| [worker\_iam\_instance\_profile\_arns](#output\_worker\_iam\_instance\_profile\_arns) | default IAM instance profile ARN for EKS worker groups | +| [worker\_iam\_instance\_profile\_names](#output\_worker\_iam\_instance\_profile\_names) | default IAM instance profile name for EKS worker groups | +| [worker\_iam\_role\_arn](#output\_worker\_iam\_role\_arn) | default IAM role ARN for EKS worker groups | +| [worker\_iam\_role\_name](#output\_worker\_iam\_role\_name) | default IAM role name for EKS worker groups | +| [worker\_security\_group\_id](#output\_worker\_security\_group\_id) | Security group ID attached to the EKS workers. | +| [workers\_asg\_arns](#output\_workers\_asg\_arns) | IDs of the autoscaling groups containing workers. | +| [workers\_asg\_names](#output\_workers\_asg\_names) | Names of the autoscaling groups containing workers. | +| [workers\_default\_ami\_id](#output\_workers\_default\_ami\_id) | ID of the default worker group AMI | +| [workers\_launch\_template\_arns](#output\_workers\_launch\_template\_arns) | ARNs of the worker launch templates. | +| [workers\_launch\_template\_ids](#output\_workers\_launch\_template\_ids) | IDs of the worker launch templates. | +| [workers\_launch\_template\_latest\_versions](#output\_workers\_launch\_template\_latest\_versions) | Latest versions of the worker launch templates. | +| [workers\_user\_data](#output\_workers\_user\_data) | User data of worker groups | diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 5cfedac88d..a1e443290a 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -1,26 +1,5 @@ -terraform { - required_version = ">= 0.12.0" -} - provider "aws" { - version = ">= 2.28.1" - region = var.region -} - -provider "random" { - version = "~> 2.1" -} - -provider "local" { - version = "~> 1.2" -} - -provider "null" { - version = "~> 2.1" -} - -provider "template" { - version = "~> 2.1" + region = var.region } data "aws_eks_cluster" "cluster" { @@ -36,7 +15,6 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.11" } data "aws_availability_zones" "available" { @@ -100,7 +78,7 @@ resource "aws_security_group" "all_worker_mgmt" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.47.0" + version = "~> 2.47" name = "test-vpc" cidr = "10.0.0.0/16" diff --git a/examples/basic/versions.tf b/examples/basic/versions.tf new file mode 100644 index 0000000000..c9f4a4d350 --- /dev/null +++ b/examples/basic/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + local = ">= 1.4" + null = ">= 2.1" + template = ">= 2.1" + random = ">= 2.1" + kubernetes = "~> 1.11" + } +} diff --git a/examples/create_false/main.tf b/examples/create_false/main.tf index 0aa61ef959..0afffcd2b4 100644 --- a/examples/create_false/main.tf +++ b/examples/create_false/main.tf @@ -13,11 +13,10 @@ data "aws_eks_cluster_auth" "cluster" { } provider "kubernetes" { - host = element(concat(data.aws_eks_cluster.cluster[*].endpoint, list("")), 0) - cluster_ca_certificate = base64decode(element(concat(data.aws_eks_cluster.cluster[*].certificate_authority.0.data, list("")), 0)) - token = element(concat(data.aws_eks_cluster_auth.cluster[*].token, list("")), 0) + host = element(concat(data.aws_eks_cluster.cluster[*].endpoint, [""]), 0) + cluster_ca_certificate = base64decode(element(concat(data.aws_eks_cluster.cluster[*].certificate_authority.0.data, [""]), 0)) + token = element(concat(data.aws_eks_cluster_auth.cluster[*].token, [""]), 0) load_config_file = false - version = "~> 1.11" } module "eks" { diff --git a/examples/create_false/versions.tf b/examples/create_false/versions.tf new file mode 100644 index 0000000000..9b73d00523 --- /dev/null +++ b/examples/create_false/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + kubernetes = "~> 1.11" + } +} diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf index 7fd0118c96..167805d185 100644 --- a/examples/fargate/main.tf +++ b/examples/fargate/main.tf @@ -3,24 +3,7 @@ terraform { } provider "aws" { - version = ">= 2.28.1" - region = var.region -} - -provider "random" { - version = "~> 2.1" -} - -provider "local" { - version = "~> 1.2" -} - -provider "null" { - version = "~> 2.1" -} - -provider "template" { - version = "~> 2.1" + region = var.region } data "aws_eks_cluster" "cluster" { @@ -36,7 +19,6 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.11" } data "aws_availability_zones" "available" { @@ -53,7 +35,7 @@ resource "random_string" "suffix" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.47.0" + version = "~> 2.47" name = "test-vpc" cidr = "172.16.0.0/16" diff --git a/examples/fargate/versions.tf b/examples/fargate/versions.tf new file mode 100644 index 0000000000..c9f4a4d350 --- /dev/null +++ b/examples/fargate/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + local = ">= 1.4" + null = ">= 2.1" + template = ">= 2.1" + random = ">= 2.1" + kubernetes = "~> 1.11" + } +} diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index de0186da8e..f881547262 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -1,22 +1,5 @@ -terraform { - required_version = ">= 0.12.0" -} - provider "aws" { - version = ">= 2.28.1" - region = var.region -} - -provider "local" { - version = "~> 1.2" -} - -provider "null" { - version = "~> 2.1" -} - -provider "template" { - version = "~> 2.1" + region = var.region } data "aws_eks_cluster" "cluster" { @@ -32,7 +15,6 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.11" } data "aws_availability_zones" "available" {} diff --git a/examples/irsa/versions.tf b/examples/irsa/versions.tf new file mode 100644 index 0000000000..c9f4a4d350 --- /dev/null +++ b/examples/irsa/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + local = ">= 1.4" + null = ">= 2.1" + template = ">= 2.1" + random = ">= 2.1" + kubernetes = "~> 1.11" + } +} diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index c17d5a9dfd..2501d6daa6 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -1,26 +1,5 @@ -terraform { - required_version = ">= 0.12.2" -} - provider "aws" { - version = ">= 2.28.1" - region = var.region -} - -provider "random" { - version = "~> 2.1" -} - -provider "local" { - version = "~> 1.2" -} - -provider "null" { - version = "~> 2.1" -} - -provider "template" { - version = "~> 2.1" + region = var.region } data "aws_eks_cluster" "cluster" { @@ -36,7 +15,6 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.11" } data "aws_availability_zones" "available" { @@ -53,7 +31,7 @@ resource "random_string" "suffix" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.47.0" + version = "~> 2.47" name = "test-vpc-lt" cidr = "10.0.0.0/16" diff --git a/examples/launch_templates/versions.tf b/examples/launch_templates/versions.tf new file mode 100644 index 0000000000..c9f4a4d350 --- /dev/null +++ b/examples/launch_templates/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + local = ">= 1.4" + null = ">= 2.1" + template = ">= 2.1" + random = ">= 2.1" + kubernetes = "~> 1.11" + } +} diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf index 004aa07d1a..95c43a38f0 100644 --- a/examples/launch_templates_with_managed_node_groups/main.tf +++ b/examples/launch_templates_with_managed_node_groups/main.tf @@ -1,26 +1,5 @@ -terraform { - required_version = ">= 0.12.9" -} - provider "aws" { - version = ">= 3.3.0" - region = var.region -} - -provider "random" { - version = "~> 2.1" -} - -provider "local" { - version = "~>1.4" -} - -provider "null" { - version = "~> 2.1" -} - -provider "template" { - version = "~> 2.1" + region = var.region } data "aws_eks_cluster" "cluster" { @@ -36,7 +15,6 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.11" } data "aws_availability_zones" "available" { @@ -53,7 +31,7 @@ resource "random_string" "suffix" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.47.0" + version = "~> 2.47" name = "test-vpc" cidr = "172.16.0.0/16" diff --git a/examples/launch_templates_with_managed_node_groups/versions.tf b/examples/launch_templates_with_managed_node_groups/versions.tf new file mode 100644 index 0000000000..c9f4a4d350 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + local = ">= 1.4" + null = ">= 2.1" + template = ">= 2.1" + random = ">= 2.1" + kubernetes = "~> 1.11" + } +} diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index 1bf66a0375..c2e910e4a5 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -1,26 +1,5 @@ -terraform { - required_version = ">= 0.12.6" -} - provider "aws" { - version = ">= 2.28.1" - region = var.region -} - -provider "random" { - version = "~> 2.1" -} - -provider "local" { - version = "~> 1.2" -} - -provider "null" { - version = "~> 2.1" -} - -provider "template" { - version = "~> 2.1" + region = var.region } data "aws_eks_cluster" "cluster" { @@ -36,7 +15,6 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.11" } data "aws_availability_zones" "available" { @@ -53,7 +31,7 @@ resource "random_string" "suffix" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.47.0" + version = "~> 2.47" name = "test-vpc" cidr = "172.16.0.0/16" diff --git a/examples/managed_node_groups/versions.tf b/examples/managed_node_groups/versions.tf new file mode 100644 index 0000000000..c9f4a4d350 --- /dev/null +++ b/examples/managed_node_groups/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + local = ">= 1.4" + null = ">= 2.1" + template = ">= 2.1" + random = ">= 2.1" + kubernetes = "~> 1.11" + } +} diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index 337219c4a7..4612700f20 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -1,26 +1,5 @@ -terraform { - required_version = ">= 0.12.0" -} - provider "aws" { - version = ">= 2.52.0" - region = var.region -} - -provider "random" { - version = "~> 2.1" -} - -provider "local" { - version = "~> 1.2" -} - -provider "null" { - version = "~> 2.1" -} - -provider "template" { - version = "~> 2.1" + region = var.region } data "aws_eks_cluster" "cluster" { @@ -36,7 +15,6 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.11" } data "aws_availability_zones" "available" { @@ -57,7 +35,7 @@ resource "aws_kms_key" "eks" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.47.0" + version = "~> 2.47" name = "test-vpc" cidr = "10.0.0.0/16" diff --git a/examples/secrets_encryption/versions.tf b/examples/secrets_encryption/versions.tf new file mode 100644 index 0000000000..c9f4a4d350 --- /dev/null +++ b/examples/secrets_encryption/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + local = ">= 1.4" + null = ">= 2.1" + template = ">= 2.1" + random = ">= 2.1" + kubernetes = "~> 1.11" + } +} diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index d2f77f76d7..67215d16a9 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -1,26 +1,5 @@ -terraform { - required_version = ">= 0.12.2" -} - provider "aws" { - version = ">= 2.28.1" - region = var.region -} - -provider "random" { - version = "~> 2.1" -} - -provider "local" { - version = "~> 1.2" -} - -provider "null" { - version = "~> 2.1" -} - -provider "template" { - version = "~> 2.1" + region = var.region } data "aws_eks_cluster" "cluster" { @@ -36,7 +15,6 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.11" } data "aws_availability_zones" "available" { @@ -53,7 +31,7 @@ resource "random_string" "suffix" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.47.0" + version = "~> 2.47" name = "test-vpc-spot" cidr = "10.0.0.0/16" diff --git a/examples/spot_instances/versions.tf b/examples/spot_instances/versions.tf new file mode 100644 index 0000000000..c9f4a4d350 --- /dev/null +++ b/examples/spot_instances/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + local = ">= 1.4" + null = ">= 2.1" + template = ">= 2.1" + random = ">= 2.1" + kubernetes = "~> 1.11" + } +} diff --git a/local.tf b/local.tf index 09d62f0df2..517fe7ab97 100644 --- a/local.tf +++ b/local.tf @@ -1,7 +1,7 @@ locals { cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id - cluster_primary_security_group_id = var.cluster_version >= 1.14 ? element(concat(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, list("")), 0) : null + cluster_primary_security_group_id = var.cluster_version >= 1.14 ? element(concat(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, [""]), 0) : null cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id diff --git a/modules/fargate/README.md b/modules/fargate/README.md index 8b4c72f25e..d31e59735b 100644 --- a/modules/fargate/README.md +++ b/modules/fargate/README.md @@ -19,38 +19,54 @@ Helper submodule to create and manage resources related to `aws_eks_fargate_prof ## Requirements -No requirements. +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | ## Providers | Name | Version | |------|---------| -| aws | n/a | +| [aws](#provider\_aws) | >= 3.22.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_eks_fargate_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_fargate_profile) | resource | +| [aws_iam_role.eks_fargate_pod](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.eks_fargate_pod](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_policy_document.eks_fargate_pod_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_role.custom_fargate_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| cluster\_name | Name of the EKS cluster. | `string` | n/a | yes | -| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | -| create\_fargate\_pod\_execution\_role | Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created. | `bool` | `true` | no | -| eks\_depends\_on | List of references to other resources this submodule depends on. | `any` | `null` | no | -| fargate\_pod\_execution\_role\_name | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | -| fargate\_profiles | Fargate profiles to create. See `fargate_profile` keys section in README.md for more details | `any` | `{}` | no | -| iam\_path | IAM roles will be created on this path. | `string` | `"/"` | no | -| iam\_policy\_arn\_prefix | IAM policy prefix with the correct AWS partition. | `string` | n/a | yes | -| permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | -| subnets | A list of subnets for the EKS Fargate profiles. | `list(string)` | `[]` | no | -| tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster. | `string` | n/a | yes | +| [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | +| [create\_fargate\_pod\_execution\_role](#input\_create\_fargate\_pod\_execution\_role) | Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created. | `bool` | `true` | no | +| [eks\_depends\_on](#input\_eks\_depends\_on) | List of references to other resources this submodule depends on. | `any` | `null` | no | +| [fargate\_pod\_execution\_role\_name](#input\_fargate\_pod\_execution\_role\_name) | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | +| [fargate\_profiles](#input\_fargate\_profiles) | Fargate profiles to create. See `fargate_profile` keys section in README.md for more details | `any` | `{}` | no | +| [iam\_path](#input\_iam\_path) | IAM roles will be created on this path. | `string` | `"/"` | no | +| [iam\_policy\_arn\_prefix](#input\_iam\_policy\_arn\_prefix) | IAM policy prefix with the correct AWS partition. | `string` | n/a | yes | +| [permissions\_boundary](#input\_permissions\_boundary) | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | +| [subnets](#input\_subnets) | A list of subnets for the EKS Fargate profiles. | `list(string)` | `[]` | no | +| [tags](#input\_tags) | A map of tags to add to all resources. | `map(string)` | `{}` | no | ## Outputs | Name | Description | |------|-------------| -| aws\_auth\_roles | Roles for use in aws-auth ConfigMap | -| fargate\_profile\_arns | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | -| fargate\_profile\_ids | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | -| iam\_role\_arn | IAM role ARN for EKS Fargate pods | -| iam\_role\_name | IAM role name for EKS Fargate pods | - +| [aws\_auth\_roles](#output\_aws\_auth\_roles) | Roles for use in aws-auth ConfigMap | +| [fargate\_profile\_arns](#output\_fargate\_profile\_arns) | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | +| [fargate\_profile\_ids](#output\_fargate\_profile\_ids) | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | +| [iam\_role\_arn](#output\_iam\_role\_arn) | IAM role ARN for EKS Fargate pods | +| [iam\_role\_name](#output\_iam\_role\_name) | IAM role name for EKS Fargate pods | diff --git a/modules/fargate/locals.tf b/modules/fargate/locals.tf index 4216953b77..18ba96403b 100644 --- a/modules/fargate/locals.tf +++ b/modules/fargate/locals.tf @@ -1,7 +1,7 @@ locals { create_eks = var.create_eks && length(var.fargate_profiles) > 0 - pod_execution_role_arn = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.arn, list("")), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.arn, list("")), 0) - pod_execution_role_name = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.name, list("")), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.name, list("")), 0) + pod_execution_role_arn = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.arn, [""]), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.arn, [""]), 0) + pod_execution_role_name = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.name, [""]), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.name, [""]), 0) fargate_profiles_expanded = { for k, v in var.fargate_profiles : k => merge( v, diff --git a/modules/fargate/versions.tf b/modules/fargate/versions.tf new file mode 100644 index 0000000000..1263d791e8 --- /dev/null +++ b/modules/fargate/versions.tf @@ -0,0 +1,7 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + } +} diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index e7a1967dc6..574be73214 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -39,33 +39,47 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In ## Requirements -No requirements. +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [random](#requirement\_random) | >= 2.1 | ## Providers | Name | Version | |------|---------| -| aws | n/a | -| random | n/a | +| [aws](#provider\_aws) | >= 3.22.0 | +| [random](#provider\_random) | >= 2.1 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_eks_node_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | +| [random_pet.node_groups](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| cluster\_name | Name of parent cluster | `string` | n/a | yes | -| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | -| default\_iam\_role\_arn | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | n/a | yes | -| ng\_depends\_on | List of references to other resources this submodule depends on | `any` | `null` | no | -| node\_groups | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | -| node\_groups\_defaults | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | n/a | yes | -| tags | A map of tags to add to all resources | `map(string)` | n/a | yes | -| workers\_group\_defaults | Workers group defaults from parent | `any` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Name of parent cluster | `string` | n/a | yes | +| [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | +| [default\_iam\_role\_arn](#input\_default\_iam\_role\_arn) | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | n/a | yes | +| [ng\_depends\_on](#input\_ng\_depends\_on) | List of references to other resources this submodule depends on | `any` | `null` | no | +| [node\_groups](#input\_node\_groups) | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | +| [node\_groups\_defaults](#input\_node\_groups\_defaults) | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | n/a | yes | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | n/a | yes | +| [workers\_group\_defaults](#input\_workers\_group\_defaults) | Workers group defaults from parent | `any` | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| aws\_auth\_roles | Roles for use in aws-auth ConfigMap | -| node\_groups | Outputs from EKS node groups. Map of maps, keyed by `var.node_groups` keys. See `aws_eks_node_group` Terraform documentation for values | - +| [aws\_auth\_roles](#output\_aws\_auth\_roles) | Roles for use in aws-auth ConfigMap | +| [node\_groups](#output\_node\_groups) | Outputs from EKS node groups. Map of maps, keyed by `var.node_groups` keys. See `aws_eks_node_group` Terraform documentation for values | diff --git a/modules/node_groups/versions.tf b/modules/node_groups/versions.tf new file mode 100644 index 0000000000..9fb30e02ba --- /dev/null +++ b/modules/node_groups/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + random = ">= 2.1" + } +} diff --git a/outputs.tf b/outputs.tf index 3cf145236d..62d75aaf0f 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,6 +1,6 @@ output "cluster_id" { description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" - value = element(concat(aws_eks_cluster.this.*.id, list("")), 0) + value = element(concat(aws_eks_cluster.this.*.id, [""]), 0) # So that calling plans wait for the cluster to be available before attempting # to use it. They will not need to duplicate this null_resource depends_on = [null_resource.wait_for_cluster] @@ -8,22 +8,22 @@ output "cluster_id" { output "cluster_arn" { description = "The Amazon Resource Name (ARN) of the cluster." - value = element(concat(aws_eks_cluster.this.*.arn, list("")), 0) + value = element(concat(aws_eks_cluster.this.*.arn, [""]), 0) } output "cluster_certificate_authority_data" { description = "Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster." - value = element(concat(aws_eks_cluster.this[*].certificate_authority[0].data, list("")), 0) + value = element(concat(aws_eks_cluster.this[*].certificate_authority[0].data, [""]), 0) } output "cluster_endpoint" { description = "The endpoint for your EKS Kubernetes API." - value = element(concat(aws_eks_cluster.this.*.endpoint, list("")), 0) + value = element(concat(aws_eks_cluster.this.*.endpoint, [""]), 0) } output "cluster_version" { description = "The Kubernetes server version for the EKS cluster." - value = element(concat(aws_eks_cluster.this[*].version, list("")), 0) + value = element(concat(aws_eks_cluster.this[*].version, [""]), 0) } output "cluster_security_group_id" { @@ -58,12 +58,12 @@ output "cluster_primary_security_group_id" { output "cloudwatch_log_group_name" { description = "Name of cloudwatch log group created" - value = element(concat(aws_cloudwatch_log_group.this[*].name, list("")), 0) + value = element(concat(aws_cloudwatch_log_group.this[*].name, [""]), 0) } output "cloudwatch_log_group_arn" { description = "Arn of cloudwatch log group created" - value = element(concat(aws_cloudwatch_log_group.this[*].arn, list("")), 0) + value = element(concat(aws_cloudwatch_log_group.this[*].arn, [""]), 0) } output "kubeconfig" { diff --git a/versions.tf b/versions.tf index a5eb7b6c67..f7aaf8d90c 100644 --- a/versions.tf +++ b/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.12.9, != 0.13.0" + required_version = ">= 0.13.1" required_providers { aws = ">= 3.22.0" diff --git a/workers.tf b/workers.tf index 8143e6a304..98bc327ef8 100644 --- a/workers.tf +++ b/workers.tf @@ -133,11 +133,11 @@ resource "aws_autoscaling_group" "workers" { ], [ for tag_key, tag_value in var.tags : - map( - "key", tag_key, - "value", tag_value, - "propagate_at_launch", "true" - ) + { + "key" = tag_key, + "value" = tag_value, + "propagate_at_launch" = "true" + } if tag_key != "Name" && !contains([for tag in lookup(var.worker_groups[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) ], lookup( diff --git a/workers_launch_template.tf b/workers_launch_template.tf index f574fcc510..3a9676cbe1 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -100,7 +100,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { dynamic "mixed_instances_policy" { iterator = item - for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? list(var.worker_groups_launch_template[count.index]) : [] + for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? [var.worker_groups_launch_template[count.index]] : [] content { instances_distribution { From eafc4582e7ecc50a45e3c283f5d8acaed9976afa Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Fri, 16 Apr 2021 10:00:28 +0200 Subject: [PATCH 275/962] Updated CHANGELOG --- CHANGELOG.md | 326 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 275 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85504e833d..0bf91321e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,26 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v15.0.0] - 2021-04-16 +BUG FIXES: +- Updated code and version requirements to work with Terraform 0.15 ([#1165](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1165)) + + ## [v14.0.0] - 2021-01-29 +DOCS: +- Update changelog generation to use custom sort with git-chglog v0.10.0 ([#1202](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1202)) +- Bump IRSA example dependencies to versions which work with TF 0.14 ([#1184](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1184)) +- Change instance type from `t2` to `t3` in examples ([#1169](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1169)) +- Fix typos in README and CONTRIBUTING ([#1167](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1167)) +- Make it more obvious that `var.cluster_iam_role_name` will allow reusing an existing IAM Role for the cluster. ([#1133](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1133)) +- Fixes typo in variables description ([#1154](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1154)) +- Fix a typo in the `aws-auth` section of the README ([#1099](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1099)) + +ENHANCEMENTS: +- Dont set -x in userdata to avoid printing sensitive informations in logs ([#1187](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1187)) + FEATURES: - Add nitro enclave support for EKS ([#1185](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1185)) - Add support for `service_ipv4_cidr` for the EKS cluster ([#1139](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1139)) @@ -22,21 +40,9 @@ FEATURES: - Add customizable Auto Scaling Group health check type ([#1118](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1118)) - Add permissions boundary to fargate execution IAM role ([#1108](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1108)) -ENHANCEMENTS: -- Dont set -x in userdata to avoid printing sensitive informations in logs ([#1187](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1187)) - BUG FIXES: - Merge tags from Fargate profiles with common tags from cluster ([#1159](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1159)) -DOCS: -- Update changelog generation to use custom sort with git-chglog v0.10.0 ([#1202](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1202)) -- Bump IRSA example dependencies to versions which work with TF 0.14 ([#1184](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1184)) -- Change instance type from `t2` to `t3` in examples ([#1169](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1169)) -- Fix typos in README and CONTRIBUTING ([#1167](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1167)) -- Make it more obvious that `var.cluster_iam_role_name` will allow reusing an existing IAM Role for the cluster. ([#1133](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1133)) -- Fixes typo in variables description ([#1154](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1154)) -- Fix a typo in the `aws-auth` section of the README ([#1099](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1099)) - BREAKING CHANGES: - To add add SPOT support for MNG, the `instance_type` is now a list and renamed as `instance_types`. This will probably rebuild existing Managed Node Groups. - The default root volume type is now `gp3` as it saves 20% and is more performant @@ -47,15 +53,15 @@ NOTES: ## [v13.2.1] - 2020-11-12 +DOCS: +- Clarify usage of both AWS-Managed Node Groups and Self-Managed Worker Groups ([#1094](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1094)) + ENHANCEMENTS: - Tags passed into worker groups should also be excluded from Launch Template tag specification ([#1095](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1095)) BUG FIXES: - Don’t add empty Roles ARN in aws-auth configmap, specifically when no Fargate profiles are specified ([#1096](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1096)) -DOCS: -- Clarify usage of both AWS-Managed Node Groups and Self-Managed Worker Groups ([#1094](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1094)) - ## [v13.2.0] - 2020-11-07 @@ -82,6 +88,10 @@ FEATURES: - Add `cloudwatch_log_group_arn` to outputs ([#1071](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1071)) - Add kubernetes standard labels to avoid manual mistakes on the managed `aws-auth` configmap ([#989](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/989)) +CI: +- Use ubuntu-latest instead of MacOS for docs checks ([#1074](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1074)) +- Fix GitHub Actions CI macOS build errors ([#1065](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1065)) + BUG FIXES: - The type of the output `cloudwatch_log_group_name` should be a string instead of a list of strings ([#1061](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1061)) - Use splat syntax to avoid errors during destroy with an empty state ([#1041](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1041)) @@ -89,10 +99,6 @@ BUG FIXES: - Set IAM Path for `cluster_elb_sl_role_creation` IAM policy ([#1045](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1045)) - Use the amazon `ImageOwnerAlias` for worker ami owner instead of owner id ([#1038](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1038)) -CI: -- Use ubuntu-latest instead of MacOS for docs checks ([#1074](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1074)) -- Fix GitHub Actions CI macOS build errors ([#1065](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1065)) - NOTES: - Managed Node Groups now support Launch Templates. The Launch Template it self is not managed by this module, so you have to create it by your self and pass it's id to this module. See docs and [`examples/launch_templates_with_managed_node_groups/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/launch_templates_with_managed_node_group) for more details. - The output `cloudwatch_log_group_name` was incorrectly returning the log group name as a list of strings. As a workaround, people were using `module.eks_cluster.cloudwatch_log_group_name[0]` but that was totally inconsistent with output name. Those users can now use `module.eks_cluster.cloudwatch_log_group_name` directly. @@ -101,21 +107,25 @@ NOTES: ## [v13.0.0] - 2020-10-06 +BUG FIXES: +- Use customer managed policy instead of inline policy for `cluster_elb_sl_role_creation` ([#1039](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1039)) +- More compatibility fixes for Terraform v0.13 and aws v3 ([#976](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/976)) +- Create `cluster_private_access` security group rules when it should ([#981](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/981)) +- Random_pet with LT workers under 0.13.0 ([#940](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/940)) + +ENHANCEMENTS: +- Make the `cpu_credits` optional for workers launch template ([#1030](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1030)) +- Update the `wait_for_cluster_cmd` logic to use `curl` if `wget` doesn't exist ([#1002](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1002)) + FEATURES: - Add `load_balancers` parameter to associate a CLB (Classic Load Balancer) to worker groups ASG ([#992](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/992)) - Dynamic Partition for IRSA to support AWS-CN Deployments ([#1028](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1028)) - Add AmazonEKSVPCResourceController to cluster policy to be able to set AWS Security Groups for pod ([#1011](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1011)) - Cluster version is now a required variable. ([#972](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/972)) -ENHANCEMENTS: -- Make the `cpu_credits` optional for workers launch template ([#1030](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1030)) -- Update the `wait_for_cluster_cmd` logic to use `curl` if `wget` doesn't exist ([#1002](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1002)) - -BUG FIXES: -- Use customer managed policy instead of inline policy for `cluster_elb_sl_role_creation` ([#1039](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1039)) -- More compatibility fixes for Terraform v0.13 and aws v3 ([#976](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/976)) -- Create `cluster_private_access` security group rules when it should ([#981](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/981)) -- Random_pet with LT workers under 0.13.0 ([#940](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/940)) +CI: +- Bump terraform pre-commit hook version and re-run terraform-docs with the latest version to fix the CI ([#1033](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1033)) +- Fix CI lint job ([#973](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/973)) DOCS: - Add important notes about the retry logic and the `wget` requirement ([#999](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/999)) @@ -124,10 +134,6 @@ DOCS: - Describe key_name is about AWS EC2 key pairs ([#970](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/970)) - Better documentation of `cluster_id` output blocking ([#955](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/955)) -CI: -- Bump terraform pre-commit hook version and re-run terraform-docs with the latest version to fix the CI ([#1033](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1033)) -- Fix CI lint job ([#973](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/973)) - BREAKING CHANGES: - Default for `cluster_endpoint_private_access_cidrs` is now `null` instead of `["0.0.0.0/0"]`. It makes the variable required when `cluster_create_endpoint_private_access_sg_rule` is set to `true`. This will force everyone who want to have a private access to set explicitly their allowed subnets for the sake of the principle of least access by default. - `cluster_version` variable is now required. @@ -144,6 +150,11 @@ need to depend on anything explicitly. ## [v12.2.0] - 2020-07-13 +DOCS: +- Update required IAM permissions list ([#936](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/936)) +- Improve FAQ on how to deploy from Windows ([#927](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/927)) +- Autoscaler X.Y version must match ([#928](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/928)) + FEATURES: - IMDSv2 metadata configuration in Launch Templates ([#938](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/938)) - Worker launch templates and configurations depend on security group rules and IAM policies ([#933](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/933)) @@ -155,11 +166,6 @@ BUG FIXES: - Get `on_demand_allocation_strategy` from `local.workers_group_defaults` when deciding to use `mixed_instances_policy` ([#908](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/908)) - Remove unnecessary conditional in private access security group ([#915](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/915)) -DOCS: -- Update required IAM permissions list ([#936](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/936)) -- Improve FAQ on how to deploy from Windows ([#927](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/927)) -- Autoscaler X.Y version must match ([#928](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/928)) - NOTES: - Addition of the IMDSv2 metadata configuration block to Launch Templates will cause a diff to be generated for existing Launch Templates on first Terraform apply. The defaults match existing behaviour. @@ -184,13 +190,6 @@ NOTES: ## [v12.0.0] - 2020-05-09 -FEATURES: -- Create kubeconfig with non-executable permissions ([#864](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/864)) -- Change EKS default version to 1.16 ([#857](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/857)) - -ENHANCEMENTS: -- Remove dependency on external template provider ([#854](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/854)) - BUG FIXES: - Fix Launch Templates error with aws 2.61.0 ([#875](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/875)) - Use splat syntax for cluster name to avoid `(known after apply)` in managed node groups ([#868](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/868)) @@ -199,20 +198,27 @@ DOCS: - Add notes for Kubernetes 1.16 ([#873](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/873)) - Remove useless template provider in examples ([#863](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/863)) +FEATURES: +- Create kubeconfig with non-executable permissions ([#864](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/864)) +- Change EKS default version to 1.16 ([#857](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/857)) + +ENHANCEMENTS: +- Remove dependency on external template provider ([#854](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/854)) + BREAKING CHANGES: - The default `cluster_version` is now 1.16. Kubernetes 1.16 includes a number of deprecated API removals, and you need to ensure your applications and add ons are updated, or workloads could fail after the upgrade is complete. For more information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). For action you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). Please set explicitly your `cluster_version` to an older EKS version until your workloads are ready for Kubernetes 1.16. ## [v11.1.0] - 2020-04-23 -FEATURES: -- Add support for EC2 principal in assume worker role policy for China ([#827](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/827)) - BUG FIXES: - Add `vpc_config.cluster_security_group` output as primary cluster security group id ([#828](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/828)) - Wrap `local.configmap_roles.groups` with tolist() to avoid panic ([#846](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/846)) - Prevent `coalescelist` null argument error when destroying worker_group_launch_templates ([#842](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/842)) +FEATURES: +- Add support for EC2 principal in assume worker role policy for China ([#827](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/827)) + ## [v11.0.0] - 2020-03-31 @@ -222,7 +228,6 @@ FEATURES: - Add interpreter option to `wait_for_cluster_cmd` ([#795](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/795)) ENHANCEMENTS: -- Require kubernetes provider >=1.11.1 ([#784](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/784)) - Use `aws_partition` to build IAM policy ARNs ([#820](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/820)) - Generate `aws-auth` configmap's roles from Object. No more string concat. ([#790](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/790)) - Add timeout to default wait_for_cluster_cmd ([#791](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/791)) @@ -237,14 +242,200 @@ BUG FIXES: - Fix git-chglog template to format changelog `Type` nicely ([#803](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/803)) - Fix git-chglog configuration ([#802](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/802)) +CI: +- Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) + TESTS: - Remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) -CI: -- Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) + + +## [v10.0.0] - 2020-03-12 + + + + +## [v9.0.0] - 2020-02-27 + +- V9.0.0 ([#752](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/752)) +- {Create,Delete}OpenIDProviderConnect` to required IAM policies ([#729](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/729)) +- GetOpenIDConnectProvider grant to docs/iam-permissions.md ([#728](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/728)) + + + +## [v8.2.0] - 2020-01-29 + + + + +## [v8.1.0] - 2020-01-17 + + + + +## [v8.0.0] - 2020-01-09 + +- [#606](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/606) ([#611](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/611)) + + + +## [v7.0.1] - 2019-11-20 + + + + +## [v7.0.0] - 2019-10-30 + + + + +## [v6.0.2] - 2019-10-07 + + + + +## [v6.0.1] - 2019-09-25 + + + + +## [v6.0.0] - 2019-09-18 + +- Correct elb tags ([#458](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/458)) + + + +## [v5.1.0] - 2019-07-30 + +- `enable_dns_hostnames = true` in examples ([#446](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/446)) +- Fix README.md worker_groups tags syntax ([#405](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/405)) + + + +## [v5.0.0] - 2019-06-19 + +- Now supporting TF 0.12!! ([#399](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/399)) + + + +## [v4.0.2] - 2019-05-08 + + + + +## [v4.0.1] - 2019-05-07 + +- Worker_group_xx vs worker_groups_xx ([#374](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/374)) + + + +## [v4.0.0] - 2019-05-07 + +- AMI ID and work user-data ([#364](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/364)) + + + +## [v3.0.0] - 2019-04-15 + +- ENI's prevent SecGrps from being destroyed on tf destroy ([#311](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/311)) + + + +## [v2.3.1] - 2019-03-26 + +- 2.3.1 ([#321](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/321)) + + + +## [v2.3.0] - 2019-03-20 + +- DescribeLaunchTemplateVersions action to worker node iam role + + + +## [v2.2.1] - 2019-02-18 + + + + +## [v2.2.0] - 2019-02-07 + + + + +## [v2.1.0] - 2019-01-16 + + + + +## [v2.0.0] - 2018-12-17 + + + + +## [v1.8.0] - 2018-12-04 + + + + +## [v1.7.0] - 2018-10-09 + +- 'aws_iam_instance_profile.workers' not found ([#141](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/141)) + + + +## [v1.6.0] - 2018-09-04 + + + + +## [v1.5.0] - 2018-08-30 + + + + +## [v1.4.0] - 2018-08-02 + +- //github.com/ozbillwang/terraform-aws-eks into [#57](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/57) + + + +## [v1.3.0] - 2018-07-11 + + + + +## [v1.2.0] - 2018-07-01 + + + + +## [v1.1.0] - 2018-06-25 + + + + +## [v1.0.0] - 2018-06-11 + + + + +## [v0.2.0] - 2018-06-08 + + + + +## [v0.1.1] - 2018-06-07 + + + + +## v0.1.0 - 2018-06-07 + -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v14.0.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.0.0...HEAD +[v15.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v14.0.0...v15.0.0 [v14.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.1...v14.0.0 [v13.2.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.0...v13.2.1 [v13.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.1.0...v13.2.0 @@ -255,3 +446,36 @@ CI: [v12.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.1.0...v12.0.0 [v11.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.0.0...v11.1.0 [v11.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v10.0.0...v11.0.0 +[v10.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...v10.0.0 +[v9.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...v9.0.0 +[v8.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...v8.2.0 +[v8.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0 +[v8.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.1...v8.0.0 +[v7.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...v7.0.1 +[v7.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v7.0.0 +[v6.0.2]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2 +[v6.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...v6.0.1 +[v6.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v6.0.0 +[v5.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...v5.1.0 +[v5.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...v5.0.0 +[v4.0.2]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.1...v4.0.2 +[v4.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1 +[v4.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v3.0.0...v4.0.0 +[v3.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.1...v3.0.0 +[v2.3.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.0...v2.3.1 +[v2.3.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.3.0 +[v2.2.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.0...v2.2.1 +[v2.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.1.0...v2.2.0 +[v2.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.0.0...v2.1.0 +[v2.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.8.0...v2.0.0 +[v1.8.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.7.0...v1.8.0 +[v1.7.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.6.0...v1.7.0 +[v1.6.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.5.0...v1.6.0 +[v1.5.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.4.0...v1.5.0 +[v1.4.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.3.0...v1.4.0 +[v1.3.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.2.0...v1.3.0 +[v1.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.1.0...v1.2.0 +[v1.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.0.0...v1.1.0 +[v1.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.2.0...v1.0.0 +[v0.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.1...v0.2.0 +[v0.1.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.0...v0.1.1 From ffdc7a79a575d0c1866bc97d3d7c804990cd66a9 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Fri, 16 Apr 2021 12:43:02 +0200 Subject: [PATCH 276/962] fix: Fixed list and map usage (#1307) --- workers_launch_template.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 3a9676cbe1..c9626f491f 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -164,7 +164,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { dynamic "launch_template" { iterator = item - for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? [] : list(var.worker_groups_launch_template[count.index]) + for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? [] : [var.worker_groups_launch_template[count.index]] content { id = aws_launch_template.workers_launch_template.*.id[count.index] @@ -209,11 +209,11 @@ resource "aws_autoscaling_group" "workers_launch_template" { ], [ for tag_key, tag_value in var.tags : - map( - "key", tag_key, - "value", tag_value, - "propagate_at_launch", "true" - ) + tomap({ + key = tag_key + value = tag_value + propagate_at_launch = "true" + }) if tag_key != "Name" && !contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) ], lookup( From 64b3916fd07f4f9d3cc01f5b00de0138b248f18e Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Fri, 16 Apr 2021 12:43:20 +0200 Subject: [PATCH 277/962] Updated CHANGELOG --- CHANGELOG.md | 228 ++------------------------------------------------- 1 file changed, 9 insertions(+), 219 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bf91321e9..aba84209d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v15.1.0] - 2021-04-16 +BUG FIXES: +- Fixed list and map usage ([#1307](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1307)) + + ## [v15.0.0] - 2021-04-16 BUG FIXES: @@ -228,6 +234,7 @@ FEATURES: - Add interpreter option to `wait_for_cluster_cmd` ([#795](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/795)) ENHANCEMENTS: +- Require kubernetes provider >=1.11.1 ([#784](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/784)) - Use `aws_partition` to build IAM policy ARNs ([#820](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/820)) - Generate `aws-auth` configmap's roles from Object. No more string concat. ([#790](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/790)) - Add timeout to default wait_for_cluster_cmd ([#791](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/791)) @@ -249,192 +256,8 @@ TESTS: - Remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) - -## [v10.0.0] - 2020-03-12 - - - - -## [v9.0.0] - 2020-02-27 - -- V9.0.0 ([#752](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/752)) -- {Create,Delete}OpenIDProviderConnect` to required IAM policies ([#729](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/729)) -- GetOpenIDConnectProvider grant to docs/iam-permissions.md ([#728](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/728)) - - - -## [v8.2.0] - 2020-01-29 - - - - -## [v8.1.0] - 2020-01-17 - - - - -## [v8.0.0] - 2020-01-09 - -- [#606](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/606) ([#611](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/611)) - - - -## [v7.0.1] - 2019-11-20 - - - - -## [v7.0.0] - 2019-10-30 - - - - -## [v6.0.2] - 2019-10-07 - - - - -## [v6.0.1] - 2019-09-25 - - - - -## [v6.0.0] - 2019-09-18 - -- Correct elb tags ([#458](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/458)) - - - -## [v5.1.0] - 2019-07-30 - -- `enable_dns_hostnames = true` in examples ([#446](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/446)) -- Fix README.md worker_groups tags syntax ([#405](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/405)) - - - -## [v5.0.0] - 2019-06-19 - -- Now supporting TF 0.12!! ([#399](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/399)) - - - -## [v4.0.2] - 2019-05-08 - - - - -## [v4.0.1] - 2019-05-07 - -- Worker_group_xx vs worker_groups_xx ([#374](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/374)) - - - -## [v4.0.0] - 2019-05-07 - -- AMI ID and work user-data ([#364](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/364)) - - - -## [v3.0.0] - 2019-04-15 - -- ENI's prevent SecGrps from being destroyed on tf destroy ([#311](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/311)) - - - -## [v2.3.1] - 2019-03-26 - -- 2.3.1 ([#321](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/321)) - - - -## [v2.3.0] - 2019-03-20 - -- DescribeLaunchTemplateVersions action to worker node iam role - - - -## [v2.2.1] - 2019-02-18 - - - - -## [v2.2.0] - 2019-02-07 - - - - -## [v2.1.0] - 2019-01-16 - - - - -## [v2.0.0] - 2018-12-17 - - - - -## [v1.8.0] - 2018-12-04 - - - - -## [v1.7.0] - 2018-10-09 - -- 'aws_iam_instance_profile.workers' not found ([#141](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/141)) - - - -## [v1.6.0] - 2018-09-04 - - - - -## [v1.5.0] - 2018-08-30 - - - - -## [v1.4.0] - 2018-08-02 - -- //github.com/ozbillwang/terraform-aws-eks into [#57](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/57) - - - -## [v1.3.0] - 2018-07-11 - - - - -## [v1.2.0] - 2018-07-01 - - - - -## [v1.1.0] - 2018-06-25 - - - - -## [v1.0.0] - 2018-06-11 - - - - -## [v0.2.0] - 2018-06-08 - - - - -## [v0.1.1] - 2018-06-07 - - - - -## v0.1.0 - 2018-06-07 - - - -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.0.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.1.0...HEAD +[v15.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.0.0...v15.1.0 [v15.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v14.0.0...v15.0.0 [v14.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.1...v14.0.0 [v13.2.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.0...v13.2.1 @@ -446,36 +269,3 @@ TESTS: [v12.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.1.0...v12.0.0 [v11.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v11.0.0...v11.1.0 [v11.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v10.0.0...v11.0.0 -[v10.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v9.0.0...v10.0.0 -[v9.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.2.0...v9.0.0 -[v8.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.1.0...v8.2.0 -[v8.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v8.0.0...v8.1.0 -[v8.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.1...v8.0.0 -[v7.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...v7.0.1 -[v7.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.2...v7.0.0 -[v6.0.2]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.1...v6.0.2 -[v6.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v6.0.0...v6.0.1 -[v6.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.1.0...v6.0.0 -[v5.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v5.0.0...v5.1.0 -[v5.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.2...v5.0.0 -[v4.0.2]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.1...v4.0.2 -[v4.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v4.0.0...v4.0.1 -[v4.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v3.0.0...v4.0.0 -[v3.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.1...v3.0.0 -[v2.3.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.3.0...v2.3.1 -[v2.3.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.1...v2.3.0 -[v2.2.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.2.0...v2.2.1 -[v2.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.1.0...v2.2.0 -[v2.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v2.0.0...v2.1.0 -[v2.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.8.0...v2.0.0 -[v1.8.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.7.0...v1.8.0 -[v1.7.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.6.0...v1.7.0 -[v1.6.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.5.0...v1.6.0 -[v1.5.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.4.0...v1.5.0 -[v1.4.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.3.0...v1.4.0 -[v1.3.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.2.0...v1.3.0 -[v1.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.1.0...v1.2.0 -[v1.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v1.0.0...v1.1.0 -[v1.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.2.0...v1.0.0 -[v0.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.1...v0.2.0 -[v0.1.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.0...v0.1.1 From 2e1651df86bd315000738cf901a4cc0586be1af3 Mon Sep 17 00:00:00 2001 From: Kevin Lefevre Date: Mon, 19 Apr 2021 22:40:37 +0200 Subject: [PATCH 278/962] feat: Create launch template for Managed Node Groups (#1138) Signed-off-by: Kevin Lefevre --- modules/node_groups/README.md | 12 ++ modules/node_groups/launchtemplate.tf | 111 ++++++++++++++++++ modules/node_groups/locals.tf | 28 +++-- modules/node_groups/node_groups.tf | 18 ++- modules/node_groups/templates/userdata.sh.tpl | 6 + modules/node_groups/variables.tf | 12 ++ node_groups.tf | 18 +-- 7 files changed, 185 insertions(+), 20 deletions(-) create mode 100644 modules/node_groups/launchtemplate.tf create mode 100644 modules/node_groups/templates/userdata.sh.tpl diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 574be73214..b7443cb326 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -23,6 +23,7 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | capacity\_type | Type of instance capacity to provision. Options are `ON_DEMAND` and `SPOT` | string | Provider default behavior | | desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` | | disk\_size | Workers' disk size | number | Provider default behavior | +| disk\_type | Workers' disk type. Require `create_launch_template` to be `true`| number | `gp3` | | iam\_role\_arn | IAM role ARN for workers | string | `var.default_iam_role_arn` | | instance\_types | Node group's instance type(s). Multiple types can be specified when `capacity_type="SPOT"`. | list | `[var.workers_group_defaults[instance_type]]` | | k8s\_labels | Kubernetes labels | map(string) | No labels applied | @@ -35,6 +36,12 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | source\_security\_group\_ids | Source security groups for remote access to workers | list(string) | If key\_name is specified: THE REMOTE ACCESS WILL BE OPENED TO THE WORLD | | subnets | Subnets to contain workers | list(string) | `var.workers_group_defaults[subnets]` | | version | Kubernetes version | string | Provider default behavior | +| create_launch_template | Create and use a default launch template | bool | `false` | +| kubelet_extra_args | This string is passed directly to kubelet if set. Useful for adding labels or taints. Require `create_launch_template` to be `true`| string | "" | +| enable_monitoring | Enables/disables detailed monitoring. Require `create_launch_template` to be `true`| bool | `true` | +| eni_delete | Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) | bool | `true` | +| public_ip | Associate a public ip address with a worker. Require `create_launch_template` to be `true`| string | `false` +| pre_userdata | userdata to pre-append to the default userdata. Require `create_launch_template` to be `true`| string | "" | ## Requirements @@ -50,6 +57,7 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.22.0 | +| [cloudinit](#provider\_cloudinit) | n/a | | [random](#provider\_random) | >= 2.1 | ## Modules @@ -61,7 +69,9 @@ No modules. | Name | Type | |------|------| | [aws_eks_node_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | +| [aws_launch_template.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | | [random_pet.node_groups](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | +| [cloudinit_config.workers_userdata](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | ## Inputs @@ -74,6 +84,8 @@ No modules. | [node\_groups](#input\_node\_groups) | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | | [node\_groups\_defaults](#input\_node\_groups\_defaults) | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | n/a | yes | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | n/a | yes | +| [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | +| [worker\_security\_group\_id](#input\_worker\_security\_group\_id) | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | | [workers\_group\_defaults](#input\_workers\_group\_defaults) | Workers group defaults from parent | `any` | n/a | yes | ## Outputs diff --git a/modules/node_groups/launchtemplate.tf b/modules/node_groups/launchtemplate.tf new file mode 100644 index 0000000000..1da04e2037 --- /dev/null +++ b/modules/node_groups/launchtemplate.tf @@ -0,0 +1,111 @@ +data "cloudinit_config" "workers_userdata" { + for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] } + gzip = false + base64_encode = true + boundary = "//" + + part { + content_type = "text/x-shellscript" + content = templatefile("${path.module}/templates/userdata.sh.tpl", + { + pre_userdata = each.value["pre_userdata"] + kubelet_extra_args = each.value["kubelet_extra_args"] + } + ) + + } +} + +# This is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) +# there are several more options one could set but you probably dont need to modify them +# you can take the default and add your custom AMI and/or custom tags +# +# Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, +# then the default user-data for bootstrapping a cluster is merged in the copy. +resource "aws_launch_template" "workers" { + for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] } + name_prefix = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) + description = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) + update_default_version = true + + block_device_mappings { + device_name = "/dev/xvda" + + ebs { + volume_size = lookup(each.value, "disk_size", null) + volume_type = lookup(each.value, "disk_type", null) + delete_on_termination = true + } + } + + instance_type = each.value["set_instance_types_on_lt"] ? element(each.value.instance_types, 0) : null + + monitoring { + enabled = lookup(each.value, "enable_monitoring", null) + } + + network_interfaces { + associate_public_ip_address = lookup(each.value, "public_ip", null) + delete_on_termination = lookup(each.value, "eni_delete", null) + security_groups = flatten([ + var.worker_security_group_id, + var.worker_additional_security_group_ids, + lookup( + each.value, + "additional_security_group_ids", + null, + ), + ]) + } + + # if you want to use a custom AMI + # image_id = var.ami_id + + # If you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then + # you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 + # + # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) + + user_data = data.cloudinit_config.workers_userdata[each.key].rendered + + key_name = lookup(each.value, "key_name", null) + + # Supplying custom tags to EKS instances is another use-case for LaunchTemplates + tag_specifications { + resource_type = "instance" + + tags = merge( + var.tags, + lookup(var.node_groups_defaults, "additional_tags", {}), + lookup(var.node_groups[each.key], "additional_tags", {}), + { + Name = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) + } + ) + } + + # Supplying custom tags to EKS instances root volumes is another use-case for LaunchTemplates. (doesnt add tags to dynamically provisioned volumes via PVC tho) + tag_specifications { + resource_type = "volume" + + tags = merge( + var.tags, + lookup(var.node_groups_defaults, "additional_tags", {}), + lookup(var.node_groups[each.key], "additional_tags", {}), + { + Name = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) + } + ) + } + + # Tag the LT itself + tags = merge( + var.tags, + lookup(var.node_groups_defaults, "additional_tags", {}), + lookup(var.node_groups[each.key], "additional_tags", {}), + ) + + lifecycle { + create_before_destroy = true + } +} diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 61c633d5db..3c510d70ab 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -2,15 +2,25 @@ locals { # Merge defaults and per-group values to make code cleaner node_groups_expanded = { for k, v in var.node_groups : k => merge( { - desired_capacity = var.workers_group_defaults["asg_desired_capacity"] - iam_role_arn = var.default_iam_role_arn - instance_types = [var.workers_group_defaults["instance_type"]] - key_name = var.workers_group_defaults["key_name"] - launch_template_id = var.workers_group_defaults["launch_template_id"] - launch_template_version = var.workers_group_defaults["launch_template_version"] - max_capacity = var.workers_group_defaults["asg_max_size"] - min_capacity = var.workers_group_defaults["asg_min_size"] - subnets = var.workers_group_defaults["subnets"] + desired_capacity = var.workers_group_defaults["asg_desired_capacity"] + iam_role_arn = var.default_iam_role_arn + instance_types = [var.workers_group_defaults["instance_type"]] + key_name = var.workers_group_defaults["key_name"] + launch_template_id = var.workers_group_defaults["launch_template_id"] + launch_template_version = var.workers_group_defaults["launch_template_version"] + set_instance_types_on_lt = false + max_capacity = var.workers_group_defaults["asg_max_size"] + min_capacity = var.workers_group_defaults["asg_min_size"] + subnets = var.workers_group_defaults["subnets"] + create_launch_template = false + kubelet_extra_args = var.workers_group_defaults["kubelet_extra_args"] + disk_size = var.workers_group_defaults["root_volume_size"] + disk_type = var.workers_group_defaults["root_volume_type"] + enable_monitoring = var.workers_group_defaults["enable_monitoring"] + eni_delete = var.workers_group_defaults["eni_delete"] + public_ip = var.workers_group_defaults["public_ip"] + pre_userdata = var.workers_group_defaults["pre_userdata"] + additional_security_group_ids = var.workers_group_defaults["additional_security_group_ids"] }, var.node_groups_defaults, v, diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 68abb094d9..8b9904cb17 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -14,13 +14,13 @@ resource "aws_eks_node_group" "workers" { } ami_type = lookup(each.value, "ami_type", null) - disk_size = lookup(each.value, "disk_size", null) - instance_types = lookup(each.value, "instance_types", null) + disk_size = each.value["launch_template_id"] != null || each.value["create_launch_template"] ? null : lookup(each.value, "disk_size", null) + instance_types = !each.value["set_instance_types_on_lt"] ? each.value["instance_types"] : null release_version = lookup(each.value, "ami_release_version", null) capacity_type = lookup(each.value, "capacity_type", null) dynamic "remote_access" { - for_each = each.value["key_name"] != "" ? [{ + for_each = each.value["key_name"] != "" && each.value["launch_template_id"] == null && !each.value["create_launch_template"] ? [{ ec2_ssh_key = each.value["key_name"] source_security_group_ids = lookup(each.value, "source_security_group_ids", []) }] : [] @@ -43,6 +43,18 @@ resource "aws_eks_node_group" "workers" { } } + dynamic "launch_template" { + for_each = each.value["launch_template_id"] == null && each.value["create_launch_template"] ? [{ + id = aws_launch_template.workers[each.key].id + version = aws_launch_template.workers[each.key].latest_version + }] : [] + + content { + id = launch_template.value["id"] + version = launch_template.value["version"] + } + } + version = lookup(each.value, "version", null) labels = merge( diff --git a/modules/node_groups/templates/userdata.sh.tpl b/modules/node_groups/templates/userdata.sh.tpl new file mode 100644 index 0000000000..3aecd0aabb --- /dev/null +++ b/modules/node_groups/templates/userdata.sh.tpl @@ -0,0 +1,6 @@ +#!/bin/bash -e + +# Allow user supplied pre userdata code +${pre_userdata} + +sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" ${kubelet_extra_args}"' /etc/eks/bootstrap.sh diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf index fc869d9d99..585beb5f91 100644 --- a/modules/node_groups/variables.tf +++ b/modules/node_groups/variables.tf @@ -19,6 +19,18 @@ variable "workers_group_defaults" { type = any } +variable "worker_security_group_id" { + description = "If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster." + type = string + default = "" +} + +variable "worker_additional_security_group_ids" { + description = "A list of additional security group ids to attach to worker instances" + type = list(string) + default = [] +} + variable "tags" { description = "A map of tags to add to all resources" type = map(string) diff --git a/node_groups.tf b/node_groups.tf index 6721f51aa2..d98979310f 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -1,12 +1,14 @@ module "node_groups" { - source = "./modules/node_groups" - create_eks = var.create_eks - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] - default_iam_role_arn = coalescelist(aws_iam_role.workers[*].arn, [""])[0] - workers_group_defaults = local.workers_group_defaults - tags = var.tags - node_groups_defaults = var.node_groups_defaults - node_groups = var.node_groups + source = "./modules/node_groups" + create_eks = var.create_eks + cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + default_iam_role_arn = coalescelist(aws_iam_role.workers[*].arn, [""])[0] + workers_group_defaults = local.workers_group_defaults + worker_security_group_id = local.worker_security_group_id + worker_additional_security_group_ids = var.worker_additional_security_group_ids + tags = var.tags + node_groups_defaults = var.node_groups_defaults + node_groups = var.node_groups # Hack to ensure ordering of resource creation. # This is a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 From 26d40d2f1dd29cee1f4c05354b4b1afcc2b38ceb Mon Sep 17 00:00:00 2001 From: Rucha Kulkarni <41280208+rucha3@users.noreply.github.com> Date: Tue, 20 Apr 2021 02:16:17 +0530 Subject: [PATCH 279/962] feat: Add support for specifying elastic inference accelerator (#1176) --- examples/launch_templates/main.tf | 7 +++++++ local.tf | 1 + workers_launch_template.tf | 12 ++++++++++++ 3 files changed, 20 insertions(+) diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 2501d6daa6..98cac244bf 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -60,5 +60,12 @@ module "eks" { asg_desired_capacity = 1 public_ip = true }, + { + name = "worker-group-3" + instance_type = "t2.large" + asg_desired_capacity = 1 + public_ip = true + elastic_inference_accelerator = "eia2.medium" + }, ] } diff --git a/local.tf b/local.tf index 517fe7ab97..f60ef1c335 100644 --- a/local.tf +++ b/local.tf @@ -94,6 +94,7 @@ locals { spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price max_instance_lifetime = 0 # Maximum number of seconds instances can run in the ASG. 0 is unlimited. + elastic_inference_accelerator = null # Type of elastic inference accelerator to be attached. Example values are eia1.medium, eia2.large, etc. } workers_group_defaults = merge( diff --git a/workers_launch_template.tf b/workers_launch_template.tf index c9626f491f..bdf07afd7b 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -290,6 +290,18 @@ resource "aws_launch_template" "workers_launch_template" { "instance_type", local.workers_group_defaults["instance_type"], ) + + dynamic "elastic_inference_accelerator" { + for_each = lookup( + var.worker_groups_launch_template[count.index], + "elastic_inference_accelerator", + local.workers_group_defaults["elastic_inference_accelerator"] + ) != null ? [lookup(var.worker_groups_launch_template[count.index], "elastic_inference_accelerator", local.workers_group_defaults["elastic_inference_accelerator"])] : [] + content { + type = elastic_inference_accelerator.value + } + } + key_name = lookup( var.worker_groups_launch_template[count.index], "key_name", From abdcfb7261016c6b6730b89f2fa85d34c1a8e3c3 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 19 Apr 2021 22:54:01 +0200 Subject: [PATCH 280/962] fix: Change back the default disk type to `gp2` (#1208) --- local.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local.tf b/local.tf index f60ef1c335..4092762316 100644 --- a/local.tf +++ b/local.tf @@ -43,7 +43,7 @@ locals { spot_price = "" # Cost of spot instance. placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". root_volume_size = "100" # root volume size of workers instances. - root_volume_type = "gp3" # root volume type of workers instances, can be "standard", "gp3", "gp2", or "io1" + root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp3' (for Launch Template), 'gp2', or 'io1' root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". root_volume_throughput = null # The amount of throughput to provision for a gp3 volume. key_name = "" # The key pair name that should be used for the instances in the autoscaling group From fb5939314a4ea38ef40a942eb8cdf6f0b54c5222 Mon Sep 17 00:00:00 2001 From: danvbloomberg <77079354+danvbloomberg@users.noreply.github.com> Date: Mon, 19 Apr 2021 14:02:08 -0700 Subject: [PATCH 281/962] fix: Add missing IAM permission for NLB with EIPs (#1226) --- cluster.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster.tf b/cluster.tf index 53a696bdee..03c30303c3 100644 --- a/cluster.tf +++ b/cluster.tf @@ -156,7 +156,8 @@ data "aws_iam_policy_document" "cluster_elb_sl_role_creation" { effect = "Allow" actions = [ "ec2:DescribeAccountAttributes", - "ec2:DescribeInternetGateways" + "ec2:DescribeInternetGateways", + "ec2:DescribeAddresses" ] resources = ["*"] } From 25494088037c9d58d71dbe4db91a57a28b5174c6 Mon Sep 17 00:00:00 2001 From: Sergiu Plotnicu <33776238+sppwf@users.noreply.github.com> Date: Tue, 20 Apr 2021 00:05:06 +0300 Subject: [PATCH 282/962] feat: Add support for ASG Warm Pools (#1310) --- local.tf | 1 + workers.tf | 10 ++++++++++ workers_launch_template.tf | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/local.tf b/local.tf index 4092762316..924f9d79e0 100644 --- a/local.tf +++ b/local.tf @@ -71,6 +71,7 @@ locals { termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. platform = "linux" # Platform of workers. either "linux" or "windows" additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults + warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device diff --git a/workers.tf b/workers.tf index 98bc327ef8..9f5ee7bd2c 100644 --- a/workers.tf +++ b/workers.tf @@ -112,6 +112,16 @@ resource "aws_autoscaling_group" "workers" { } } + dynamic "warm_pool" { + for_each = lookup(var.worker_groups[count.index], "warm_pool", null) != null ? [lookup(var.worker_groups[count.index], "warm_pool")] : [] + + content { + pool_state = lookup(warm_pool.value, "pool_state", null) + min_size = lookup(warm_pool.value, "min_size", null) + max_group_prepared_capacity = lookup(warm_pool.value, "max_group_prepared_capacity", null) + } + } + dynamic "tag" { for_each = concat( [ diff --git a/workers_launch_template.tf b/workers_launch_template.tf index bdf07afd7b..a7b5f62a40 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -189,6 +189,16 @@ resource "aws_autoscaling_group" "workers_launch_template" { } } + dynamic "warm_pool" { + for_each = lookup(var.worker_groups_launch_template[count.index], "warm_pool", null) != null ? [lookup(var.worker_groups_launch_template[count.index], "warm_pool")] : [] + + content { + pool_state = lookup(warm_pool.value, "pool_state", null) + min_size = lookup(warm_pool.value, "min_size", null) + max_group_prepared_capacity = lookup(warm_pool.value, "max_group_prepared_capacity", null) + } + } + dynamic "tag" { for_each = concat( [ From fc6852d63a870ce4a6de838ecb5a229699987e6f Mon Sep 17 00:00:00 2001 From: Kamil Aliev Date: Tue, 20 Apr 2021 00:10:25 +0300 Subject: [PATCH 283/962] fix: Add mission metadata block for launch configuration (#1301) --- workers.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/workers.tf b/workers.tf index 9f5ee7bd2c..35132b2675 100644 --- a/workers.tf +++ b/workers.tf @@ -234,6 +234,24 @@ resource "aws_launch_configuration" "workers" { local.workers_group_defaults["placement_tenancy"], ) + metadata_options { + http_endpoint = lookup( + var.worker_groups[count.index], + "metadata_http_endpoint", + local.workers_group_defaults["metadata_http_endpoint"], + ) + http_tokens = lookup( + var.worker_groups[count.index], + "metadata_http_tokens", + local.workers_group_defaults["metadata_http_tokens"], + ) + http_put_response_hop_limit = lookup( + var.worker_groups[count.index], + "metadata_http_put_response_hop_limit", + local.workers_group_defaults["metadata_http_put_response_hop_limit"], + ) + } + root_block_device { encrypted = lookup( var.worker_groups[count.index], From b752b1d5172936a912b8175e03adcf3d79af2b12 Mon Sep 17 00:00:00 2001 From: Tom Stewart Date: Mon, 19 Apr 2021 17:16:17 -0400 Subject: [PATCH 284/962] docs: update helm instructions for irsa example (#1251) --- examples/irsa/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/irsa/README.md b/examples/irsa/README.md index 3cbe5e67e9..53dc500dab 100644 --- a/examples/irsa/README.md +++ b/examples/irsa/README.md @@ -30,7 +30,7 @@ Install the chart using the provided values file: ``` $ helm repo add autoscaler https://kubernetes.github.io/autoscaler $ helm repo update -$ helm install cluster-autoscaler --namespace kube-system autoscaler/cluster-autoscaler-chart --values=cluster-autoscaler-chart-values.yaml +$ helm install cluster-autoscaler --namespace kube-system autoscaler/cluster-autoscaler --values cluster-autoscaler-chart-values.yaml ``` ## Verify From f8bdea11ab13e46e15b3a4d52dc579e0f2ec1623 Mon Sep 17 00:00:00 2001 From: Gustavo Cavalieri Fernandes Date: Mon, 19 Apr 2021 18:25:44 -0300 Subject: [PATCH 285/962] fix: limit cluster_name when creating fargate IAM Role (#1270) --- modules/fargate/fargate.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/fargate/fargate.tf b/modules/fargate/fargate.tf index 43f616315a..e88adb63bb 100644 --- a/modules/fargate/fargate.tf +++ b/modules/fargate/fargate.tf @@ -1,6 +1,6 @@ resource "aws_iam_role" "eks_fargate_pod" { count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 - name_prefix = format("%s-fargate", var.cluster_name) + name_prefix = format("%s-fargate", substr(var.cluster_name, 0, 24)) assume_role_policy = data.aws_iam_policy_document.eks_fargate_pod_assume_role[0].json permissions_boundary = var.permissions_boundary tags = var.tags From 81bc7a2dd1f6865f18b874d99f95d1ec1833d1f8 Mon Sep 17 00:00:00 2001 From: Xavier Fernandez Date: Tue, 20 Apr 2021 10:26:39 +0200 Subject: [PATCH 286/962] feat: Allow to specify the managed cluster IAM role name (#1199) --- README.md | 2 +- cluster.tf | 3 ++- variables.tf | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4687b2e829..33e668cb9a 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [cluster\_endpoint\_private\_access\_cidrs](#input\_cluster\_endpoint\_private\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS private API server endpoint. | `list(string)` | `null` | no | | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | -| [cluster\_iam\_role\_name](#input\_cluster\_iam\_role\_name) | IAM role name for the cluster. Only applicable if manage\_cluster\_iam\_resources is set to false. Set this to reuse an existing IAM role. | `string` | `""` | no | +| [cluster\_iam\_role\_name](#input\_cluster\_iam\_role\_name) | IAM role name for the cluster. If manage\_cluster\_iam\_resources is set to false, set this to reuse an existing IAM role. If manage\_cluster\_iam\_resources is set to true, set this to force the created role name. | `string` | `""` | no | | [cluster\_log\_kms\_key\_id](#input\_cluster\_log\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `""` | no | | [cluster\_log\_retention\_in\_days](#input\_cluster\_log\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | diff --git a/cluster.tf b/cluster.tf index 03c30303c3..4047a77658 100644 --- a/cluster.tf +++ b/cluster.tf @@ -118,7 +118,8 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { resource "aws_iam_role" "cluster" { count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 - name_prefix = var.cluster_name + name_prefix = var.cluster_iam_role_name != "" ? null : var.cluster_name + name = var.cluster_iam_role_name != "" ? var.cluster_iam_role_name : null assume_role_policy = data.aws_iam_policy_document.cluster_assume_role_policy.json permissions_boundary = var.permissions_boundary path = var.iam_path diff --git a/variables.tf b/variables.tf index e49c57ca49..bcec29b2f8 100644 --- a/variables.tf +++ b/variables.tf @@ -290,7 +290,7 @@ variable "manage_cluster_iam_resources" { } variable "cluster_iam_role_name" { - description = "IAM role name for the cluster. Only applicable if manage_cluster_iam_resources is set to false. Set this to reuse an existing IAM role." + description = "IAM role name for the cluster. If manage_cluster_iam_resources is set to false, set this to reuse an existing IAM role. If manage_cluster_iam_resources is set to true, set this to force the created role name." type = string default = "" } From 2a78efd082a452641f766fb0c2205009bb25a0ab Mon Sep 17 00:00:00 2001 From: Matt Dainty Date: Tue, 20 Apr 2021 10:13:14 +0100 Subject: [PATCH 287/962] feat: Allow to override cluster and workers egress CIDRs (#1237) --- README.md | 2 ++ cluster.tf | 2 +- variables.tf | 12 ++++++++++++ workers.tf | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 33e668cb9a..42b1279612 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [cluster\_create\_security\_group](#input\_cluster\_create\_security\_group) | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | | [cluster\_create\_timeout](#input\_cluster\_create\_timeout) | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | | [cluster\_delete\_timeout](#input\_cluster\_delete\_timeout) | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | +| [cluster\_egress\_cidrs](#input\_cluster\_egress\_cidrs) | List of CIDR blocks that are permitted for cluster egress traffic. | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | | [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster. See examples/secrets\_encryption/main.tf for example format |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | | [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | @@ -287,6 +288,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [worker\_security\_group\_id](#input\_worker\_security\_group\_id) | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | | [worker\_sg\_ingress\_from\_port](#input\_worker\_sg\_ingress\_from\_port) | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | `number` | `1025` | no | | [workers\_additional\_policies](#input\_workers\_additional\_policies) | Additional policies to be added to workers | `list(string)` | `[]` | no | +| [workers\_egress\_cidrs](#input\_workers\_egress\_cidrs) | List of CIDR blocks that are permitted for workers egress traffic. | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [workers\_group\_defaults](#input\_workers\_group\_defaults) | Override default values for target groups. See workers\_group\_defaults\_defaults in local.tf for valid keys. | `any` | `{}` | no | | [workers\_role\_name](#input\_workers\_role\_name) | User defined workers role name. | `string` | `""` | no | | [write\_kubeconfig](#input\_write\_kubeconfig) | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | `bool` | `true` | no | diff --git a/cluster.tf b/cluster.tf index 4047a77658..d8ffa66a1e 100644 --- a/cluster.tf +++ b/cluster.tf @@ -99,7 +99,7 @@ resource "aws_security_group_rule" "cluster_egress_internet" { description = "Allow cluster egress access to the Internet." protocol = "-1" security_group_id = local.cluster_security_group_id - cidr_blocks = ["0.0.0.0/0"] + cidr_blocks = var.cluster_egress_cidrs from_port = 0 to_port = 0 type = "egress" diff --git a/variables.tf b/variables.tf index bcec29b2f8..6f904b6354 100644 --- a/variables.tf +++ b/variables.tf @@ -375,3 +375,15 @@ variable "cluster_service_ipv4_cidr" { type = string default = null } + +variable "cluster_egress_cidrs" { + description = "List of CIDR blocks that are permitted for cluster egress traffic." + type = list(string) + default = ["0.0.0.0/0"] +} + +variable "workers_egress_cidrs" { + description = "List of CIDR blocks that are permitted for workers egress traffic." + type = list(string) + default = ["0.0.0.0/0"] +} diff --git a/workers.tf b/workers.tf index 35132b2675..58fa3b23e2 100644 --- a/workers.tf +++ b/workers.tf @@ -360,7 +360,7 @@ resource "aws_security_group_rule" "workers_egress_internet" { description = "Allow nodes all egress to the Internet." protocol = "-1" security_group_id = local.worker_security_group_id - cidr_blocks = ["0.0.0.0/0"] + cidr_blocks = var.workers_egress_cidrs from_port = 0 to_port = 0 type = "egress" From c12c1864321ba47408e146179958322f24294581 Mon Sep 17 00:00:00 2001 From: Ivan Sukhomlyn Date: Tue, 4 May 2021 22:55:49 +0300 Subject: [PATCH 288/962] feat: Add tags on additional IAM resources like IAM policies, instance profile, OIDC provider (#1321) --- README.md | 4 ++-- cluster.tf | 1 + docs/autoscaling.md | 1 + irsa.tf | 7 +++++++ versions.tf | 2 +- workers.tf | 1 + workers_launch_template.tf | 1 + 7 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 42b1279612..e05d8f565c 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | +| [aws](#requirement\_aws) | >= 3.35.0 | | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [null](#requirement\_null) | >= 2.1 | @@ -156,7 +156,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | +| [aws](#provider\_aws) | >= 3.35.0 | | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | | [null](#provider\_null) | >= 2.1 | diff --git a/cluster.tf b/cluster.tf index d8ffa66a1e..4306db3073 100644 --- a/cluster.tf +++ b/cluster.tf @@ -170,6 +170,7 @@ resource "aws_iam_policy" "cluster_elb_sl_role_creation" { description = "Permissions for EKS to create AWSServiceRoleForElasticLoadBalancing service-linked role" policy = data.aws_iam_policy_document.cluster_elb_sl_role_creation[0].json path = var.iam_path + tags = var.tags } resource "aws_iam_role_policy_attachment" "cluster_elb_sl_role_creation" { diff --git a/docs/autoscaling.md b/docs/autoscaling.md index 28d737f418..c5ba615433 100644 --- a/docs/autoscaling.md +++ b/docs/autoscaling.md @@ -21,6 +21,7 @@ resource "aws_iam_policy" "worker_autoscaling" { description = "EKS worker node autoscaling policy for cluster ${module.my_cluster.cluster_id}" policy = data.aws_iam_policy_document.worker_autoscaling.json path = var.iam_path + tags = var.tags } data "aws_iam_policy_document" "worker_autoscaling" { diff --git a/irsa.tf b/irsa.tf index 63a19871b5..9c5d653a25 100644 --- a/irsa.tf +++ b/irsa.tf @@ -12,4 +12,11 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { client_id_list = [local.sts_principal] thumbprint_list = [var.eks_oidc_root_ca_thumbprint] url = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0] + + tags = merge( + { + Name = "${var.cluster_name}-eks-irsa" + }, + var.tags + ) } diff --git a/versions.tf b/versions.tf index f7aaf8d90c..f2e5431970 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" + aws = ">= 3.35.0" local = ">= 1.4" null = ">= 2.1" template = ">= 2.1" diff --git a/workers.tf b/workers.tf index 58fa3b23e2..2bd75a27ff 100644 --- a/workers.tf +++ b/workers.tf @@ -453,6 +453,7 @@ resource "aws_iam_instance_profile" "workers" { ) path = var.iam_path + tags = var.tags lifecycle { create_before_destroy = true diff --git a/workers_launch_template.tf b/workers_launch_template.tf index a7b5f62a40..3b16ff163a 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -563,6 +563,7 @@ resource "aws_iam_instance_profile" "workers_launch_template" { local.default_iam_role_id, ) path = var.iam_path + tags = var.tags lifecycle { create_before_destroy = true From 3ecdc20ce6c14d3e5c8d974b860f705ee67cfd9f Mon Sep 17 00:00:00 2001 From: Neven Miculinic Date: Tue, 4 May 2021 21:58:44 +0200 Subject: [PATCH 289/962] fix: Replace `list` with `tolist` function for working with terraform v0.15.0 (#1317) --- workers_launch_template.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 3b16ff163a..1602437e55 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -385,7 +385,7 @@ resource "aws_launch_template" "workers_launch_template" { } dynamic "instance_market_options" { - for_each = lookup(var.worker_groups_launch_template[count.index], "market_type", null) == null ? [] : list(lookup(var.worker_groups_launch_template[count.index], "market_type", null)) + for_each = lookup(var.worker_groups_launch_template[count.index], "market_type", null) == null ? [] : tolist([lookup(var.worker_groups_launch_template[count.index], "market_type", null)]) content { market_type = instance_market_options.value } From 65ba9de0d833c4b33f4e489befd27d9aa4afe725 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 4 May 2021 23:08:09 +0200 Subject: [PATCH 290/962] chore: update changelog (#1334) --- CHANGELOG.md | 124 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 73 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aba84209d1..4b3c9f803a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,27 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v15.2.0] - 2021-05-04 +FEATURES: +- Add tags on additional IAM resources like IAM policies, instance profile, OIDC provider ([#1321](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1321)) +- Allow to override cluster and workers egress CIDRs ([#1237](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1237)) +- Allow to specify the managed cluster IAM role name ([#1199](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1199)) +- Add support for ASG Warm Pools ([#1310](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1310)) +- Add support for specifying elastic inference accelerator ([#1176](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1176)) +- Create launch template for Managed Node Groups ([#1138](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1138)) + +BUG FIXES: +- Replace `list` with `tolist` function for working with terraform v0.15.0 ([#1317](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1317)) +- Limit cluster_name when creating fargate IAM Role ([#1270](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1270)) +- Add mission metadata block for launch configuration ([#1301](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1301)) +- Add missing IAM permission for NLB with EIPs ([#1226](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1226)) +- Change back the default disk type to `gp2` ([#1208](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1208)) + +DOCS: +- Update helm instructions for irsa example ([#1251](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1251)) + + ## [v15.1.0] - 2021-04-16 BUG FIXES: @@ -24,18 +45,6 @@ BUG FIXES: ## [v14.0.0] - 2021-01-29 -DOCS: -- Update changelog generation to use custom sort with git-chglog v0.10.0 ([#1202](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1202)) -- Bump IRSA example dependencies to versions which work with TF 0.14 ([#1184](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1184)) -- Change instance type from `t2` to `t3` in examples ([#1169](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1169)) -- Fix typos in README and CONTRIBUTING ([#1167](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1167)) -- Make it more obvious that `var.cluster_iam_role_name` will allow reusing an existing IAM Role for the cluster. ([#1133](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1133)) -- Fixes typo in variables description ([#1154](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1154)) -- Fix a typo in the `aws-auth` section of the README ([#1099](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1099)) - -ENHANCEMENTS: -- Dont set -x in userdata to avoid printing sensitive informations in logs ([#1187](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1187)) - FEATURES: - Add nitro enclave support for EKS ([#1185](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1185)) - Add support for `service_ipv4_cidr` for the EKS cluster ([#1139](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1139)) @@ -46,9 +55,21 @@ FEATURES: - Add customizable Auto Scaling Group health check type ([#1118](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1118)) - Add permissions boundary to fargate execution IAM role ([#1108](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1108)) +ENHANCEMENTS: +- Dont set -x in userdata to avoid printing sensitive informations in logs ([#1187](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1187)) + BUG FIXES: - Merge tags from Fargate profiles with common tags from cluster ([#1159](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1159)) +DOCS: +- Update changelog generation to use custom sort with git-chglog v0.10.0 ([#1202](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1202)) +- Bump IRSA example dependencies to versions which work with TF 0.14 ([#1184](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1184)) +- Change instance type from `t2` to `t3` in examples ([#1169](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1169)) +- Fix typos in README and CONTRIBUTING ([#1167](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1167)) +- Make it more obvious that `var.cluster_iam_role_name` will allow reusing an existing IAM Role for the cluster. ([#1133](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1133)) +- Fixes typo in variables description ([#1154](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1154)) +- Fix a typo in the `aws-auth` section of the README ([#1099](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1099)) + BREAKING CHANGES: - To add add SPOT support for MNG, the `instance_type` is now a list and renamed as `instance_types`. This will probably rebuild existing Managed Node Groups. - The default root volume type is now `gp3` as it saves 20% and is more performant @@ -59,15 +80,15 @@ NOTES: ## [v13.2.1] - 2020-11-12 -DOCS: -- Clarify usage of both AWS-Managed Node Groups and Self-Managed Worker Groups ([#1094](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1094)) - ENHANCEMENTS: - Tags passed into worker groups should also be excluded from Launch Template tag specification ([#1095](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1095)) BUG FIXES: - Don’t add empty Roles ARN in aws-auth configmap, specifically when no Fargate profiles are specified ([#1096](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1096)) +DOCS: +- Clarify usage of both AWS-Managed Node Groups and Self-Managed Worker Groups ([#1094](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1094)) + ## [v13.2.0] - 2020-11-07 @@ -94,10 +115,6 @@ FEATURES: - Add `cloudwatch_log_group_arn` to outputs ([#1071](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1071)) - Add kubernetes standard labels to avoid manual mistakes on the managed `aws-auth` configmap ([#989](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/989)) -CI: -- Use ubuntu-latest instead of MacOS for docs checks ([#1074](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1074)) -- Fix GitHub Actions CI macOS build errors ([#1065](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1065)) - BUG FIXES: - The type of the output `cloudwatch_log_group_name` should be a string instead of a list of strings ([#1061](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1061)) - Use splat syntax to avoid errors during destroy with an empty state ([#1041](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1041)) @@ -105,6 +122,10 @@ BUG FIXES: - Set IAM Path for `cluster_elb_sl_role_creation` IAM policy ([#1045](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1045)) - Use the amazon `ImageOwnerAlias` for worker ami owner instead of owner id ([#1038](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1038)) +CI: +- Use ubuntu-latest instead of MacOS for docs checks ([#1074](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1074)) +- Fix GitHub Actions CI macOS build errors ([#1065](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1065)) + NOTES: - Managed Node Groups now support Launch Templates. The Launch Template it self is not managed by this module, so you have to create it by your self and pass it's id to this module. See docs and [`examples/launch_templates_with_managed_node_groups/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/launch_templates_with_managed_node_group) for more details. - The output `cloudwatch_log_group_name` was incorrectly returning the log group name as a list of strings. As a workaround, people were using `module.eks_cluster.cloudwatch_log_group_name[0]` but that was totally inconsistent with output name. Those users can now use `module.eks_cluster.cloudwatch_log_group_name` directly. @@ -113,25 +134,21 @@ NOTES: ## [v13.0.0] - 2020-10-06 -BUG FIXES: -- Use customer managed policy instead of inline policy for `cluster_elb_sl_role_creation` ([#1039](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1039)) -- More compatibility fixes for Terraform v0.13 and aws v3 ([#976](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/976)) -- Create `cluster_private_access` security group rules when it should ([#981](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/981)) -- Random_pet with LT workers under 0.13.0 ([#940](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/940)) - -ENHANCEMENTS: -- Make the `cpu_credits` optional for workers launch template ([#1030](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1030)) -- Update the `wait_for_cluster_cmd` logic to use `curl` if `wget` doesn't exist ([#1002](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1002)) - FEATURES: - Add `load_balancers` parameter to associate a CLB (Classic Load Balancer) to worker groups ASG ([#992](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/992)) - Dynamic Partition for IRSA to support AWS-CN Deployments ([#1028](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1028)) - Add AmazonEKSVPCResourceController to cluster policy to be able to set AWS Security Groups for pod ([#1011](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1011)) - Cluster version is now a required variable. ([#972](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/972)) -CI: -- Bump terraform pre-commit hook version and re-run terraform-docs with the latest version to fix the CI ([#1033](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1033)) -- Fix CI lint job ([#973](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/973)) +ENHANCEMENTS: +- Make the `cpu_credits` optional for workers launch template ([#1030](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1030)) +- Update the `wait_for_cluster_cmd` logic to use `curl` if `wget` doesn't exist ([#1002](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1002)) + +BUG FIXES: +- Use customer managed policy instead of inline policy for `cluster_elb_sl_role_creation` ([#1039](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1039)) +- More compatibility fixes for Terraform v0.13 and aws v3 ([#976](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/976)) +- Create `cluster_private_access` security group rules when it should ([#981](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/981)) +- Random_pet with LT workers under 0.13.0 ([#940](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/940)) DOCS: - Add important notes about the retry logic and the `wget` requirement ([#999](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/999)) @@ -140,6 +157,10 @@ DOCS: - Describe key_name is about AWS EC2 key pairs ([#970](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/970)) - Better documentation of `cluster_id` output blocking ([#955](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/955)) +CI: +- Bump terraform pre-commit hook version and re-run terraform-docs with the latest version to fix the CI ([#1033](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1033)) +- Fix CI lint job ([#973](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/973)) + BREAKING CHANGES: - Default for `cluster_endpoint_private_access_cidrs` is now `null` instead of `["0.0.0.0/0"]`. It makes the variable required when `cluster_create_endpoint_private_access_sg_rule` is set to `true`. This will force everyone who want to have a private access to set explicitly their allowed subnets for the sake of the principle of least access by default. - `cluster_version` variable is now required. @@ -156,11 +177,6 @@ need to depend on anything explicitly. ## [v12.2.0] - 2020-07-13 -DOCS: -- Update required IAM permissions list ([#936](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/936)) -- Improve FAQ on how to deploy from Windows ([#927](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/927)) -- Autoscaler X.Y version must match ([#928](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/928)) - FEATURES: - IMDSv2 metadata configuration in Launch Templates ([#938](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/938)) - Worker launch templates and configurations depend on security group rules and IAM policies ([#933](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/933)) @@ -172,6 +188,11 @@ BUG FIXES: - Get `on_demand_allocation_strategy` from `local.workers_group_defaults` when deciding to use `mixed_instances_policy` ([#908](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/908)) - Remove unnecessary conditional in private access security group ([#915](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/915)) +DOCS: +- Update required IAM permissions list ([#936](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/936)) +- Improve FAQ on how to deploy from Windows ([#927](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/927)) +- Autoscaler X.Y version must match ([#928](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/928)) + NOTES: - Addition of the IMDSv2 metadata configuration block to Launch Templates will cause a diff to be generated for existing Launch Templates on first Terraform apply. The defaults match existing behaviour. @@ -196,6 +217,13 @@ NOTES: ## [v12.0.0] - 2020-05-09 +FEATURES: +- Create kubeconfig with non-executable permissions ([#864](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/864)) +- Change EKS default version to 1.16 ([#857](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/857)) + +ENHANCEMENTS: +- Remove dependency on external template provider ([#854](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/854)) + BUG FIXES: - Fix Launch Templates error with aws 2.61.0 ([#875](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/875)) - Use splat syntax for cluster name to avoid `(known after apply)` in managed node groups ([#868](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/868)) @@ -204,27 +232,20 @@ DOCS: - Add notes for Kubernetes 1.16 ([#873](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/873)) - Remove useless template provider in examples ([#863](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/863)) -FEATURES: -- Create kubeconfig with non-executable permissions ([#864](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/864)) -- Change EKS default version to 1.16 ([#857](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/857)) - -ENHANCEMENTS: -- Remove dependency on external template provider ([#854](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/854)) - BREAKING CHANGES: - The default `cluster_version` is now 1.16. Kubernetes 1.16 includes a number of deprecated API removals, and you need to ensure your applications and add ons are updated, or workloads could fail after the upgrade is complete. For more information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). For action you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). Please set explicitly your `cluster_version` to an older EKS version until your workloads are ready for Kubernetes 1.16. ## [v11.1.0] - 2020-04-23 +FEATURES: +- Add support for EC2 principal in assume worker role policy for China ([#827](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/827)) + BUG FIXES: - Add `vpc_config.cluster_security_group` output as primary cluster security group id ([#828](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/828)) - Wrap `local.configmap_roles.groups` with tolist() to avoid panic ([#846](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/846)) - Prevent `coalescelist` null argument error when destroying worker_group_launch_templates ([#842](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/842)) -FEATURES: -- Add support for EC2 principal in assume worker role policy for China ([#827](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/827)) - ## [v11.0.0] - 2020-03-31 @@ -249,14 +270,15 @@ BUG FIXES: - Fix git-chglog template to format changelog `Type` nicely ([#803](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/803)) - Fix git-chglog configuration ([#802](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/802)) -CI: -- Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) - TESTS: - Remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) +CI: +- Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) + -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.1.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.2.0...HEAD +[v15.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.1.0...v15.2.0 [v15.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.0.0...v15.1.0 [v15.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v14.0.0...v15.0.0 [v14.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v13.2.1...v14.0.0 From 5db15e3d6dd96fb9bcefc59debafa0d86e5110a2 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 4 May 2021 23:30:06 +0200 Subject: [PATCH 291/962] ci: Bump terraform-docs to 0.13 (#1335) --- .github/workflows/pre-commit.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 59cd0a8967..ab9aef239c 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -38,7 +38,7 @@ jobs: uses: actions/setup-python@v2 - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.0.1 + uses: clowdhaus/terraform-min-max@v1.0.2 with: directory: ${{ matrix.directory }} - name: Install Terraform v${{ steps.minMax.outputs.minVersion }} @@ -50,14 +50,11 @@ jobs: - name: Execute pre-commit # Run only validate pre-commit check on min version supported if: ${{ matrix.directory != '.' }} - run: - pre-commit run terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/* + run: pre-commit run terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/* - name: Execute pre-commit # Run only validate pre-commit check on min version supported if: ${{ matrix.directory == '.' }} - run: - pre-commit run terraform_validate --color=always --show-diff-on-failure --files $(ls *.tf) - + run: pre-commit run terraform_validate --color=always --show-diff-on-failure --files $(ls *.tf) # Max Terraform version getBaseVersion: @@ -68,7 +65,7 @@ jobs: uses: actions/checkout@v2 - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.0.1 + uses: clowdhaus/terraform-min-max@v1.0.2 outputs: minVersion: ${{ steps.minMax.outputs.minVersion }} maxVersion: ${{ steps.minMax.outputs.maxVersion }} @@ -94,7 +91,7 @@ jobs: - name: Install pre-commit dependencies run: | pip install pre-commit - curl -L "$(curl -s https://api.github.com/repos/terraform-docs/terraform-docs/releases/latest | grep -o -E "https://.+?-v0.12\..+?-linux-amd64" | head -n1)" > terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ + curl -Lo ./terraform-docs.tar.gz https://github.com/terraform-docs/terraform-docs/releases/download/v0.13.0/terraform-docs-v0.13.0-$(uname)-amd64.tar.gz && tar -xzf terraform-docs.tar.gz && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/ - name: Execute pre-commit # Run all pre-commit checks on max version supported From 781f673295dd69d24c85d10b47cb03507f394402 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 6 May 2021 13:09:31 +0200 Subject: [PATCH 292/962] fix: Bump AWS provider version to add Warm Pool support (#1340) --- versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.tf b/versions.tf index f2e5431970..6d70b06380 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.35.0" + aws = ">= 3.37.0" local = ">= 1.4" null = ">= 2.1" template = ">= 2.1" From e3bf48bba22dcab5ca5c7962651af4ef6a43e3f5 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 17 May 2021 10:11:36 +0200 Subject: [PATCH 293/962] feat: Replace the local-exec script with a http datasource for waiting cluster (#1339) NOTES: Using the [terraform-aws-modules/http](https://registry.terraform.io/providers/terraform-aws-modules/http/latest) provider is a more platform agnostic way to wait for the cluster availability than using a local-exec. With this change we're able to provision EKS clusters and manage the `aws_auth` configmap while still using the `hashicorp/tfc-agent` docker image. --- README.md | 24 +++++++++--------------- aws_auth.tf | 4 ++-- cluster.tf | 19 ++++--------------- docs/faq.md | 14 +------------- outputs.tf | 21 +++++++++++++++------ variables.tf | 12 ------------ versions.tf | 5 ++++- 7 files changed, 35 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index e05d8f565c..87b7ab085c 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,7 @@ You also need to ensure your applications and add ons are updated, or workloads An example of harming update was the removal of several commonly used, but deprecated APIs, in Kubernetes 1.16. More information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). -By default, this module manages the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready, we implemented a retry logic with an `local-exec` provisioner and `wget` (by default) with failover to `curl`. - -**If you want to manage your `aws-auth` configmap, ensure you have `wget` (or `curl`) and `/bin/sh` installed where you're running Terraform or set `wait_for_cluster_cmd` and `wait_for_cluster_interpreter` to match your needs.** - -For windows users, please read the following [doc](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#deploying-from-windows-binsh-file-does-not-exist). +By default, this module manages the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready. We implemented a "retry" logic with a fork of the http provider https://github.com/terraform-aws-modules/terraform-provider-http. This fork adds the support of a self-signed CA certificate. The original PR can be found at https://github.com/hashicorp/terraform-provider-http/pull/29. ## Usage example @@ -145,10 +141,10 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.35.0 | +| [aws](#requirement\_aws) | >= 3.37.0 | +| [http](#requirement\_http) | >= 2.2.0 | | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | -| [null](#requirement\_null) | >= 2.1 | | [random](#requirement\_random) | >= 2.1 | | [template](#requirement\_template) | >= 2.1 | @@ -156,10 +152,10 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.35.0 | +| [aws](#provider\_aws) | >= 3.37.0 | +| [http](#provider\_http) | >= 2.2.0 | | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | -| [null](#provider\_null) | >= 2.1 | | [random](#provider\_random) | >= 2.1 | | [template](#provider\_template) | >= 2.1 | @@ -208,7 +204,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [aws_security_group_rule.workers_ingress_self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | | [local_file.kubeconfig](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [null_resource.wait_for_cluster](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [random_pet.workers](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | | [random_pet.workers_launch_template](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | | [aws_ami.eks_worker](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | @@ -221,6 +216,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [aws_iam_policy_document.workers_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_role.custom_cluster_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [http_http.wait_for_cluster](https://registry.terraform.io/providers/terraform-aws-modules/http/latest/docs/data-sources/http) | data source | | [template_file.launch_template_userdata](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | | [template_file.userdata](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | @@ -273,8 +269,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [subnets](#input\_subnets) | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | [tags](#input\_tags) | A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | | [vpc\_id](#input\_vpc\_id) | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | -| [wait\_for\_cluster\_cmd](#input\_wait\_for\_cluster\_cmd) | Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT | `string` | `"for i in `seq 1 60`; do if `command -v wget > /dev/null`; then wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; else curl -k -s $ENDPOINT/healthz >/dev/null && exit 0 || true;fi; sleep 5; done; echo TIMEOUT && exit 1"` | no | -| [wait\_for\_cluster\_interpreter](#input\_wait\_for\_cluster\_interpreter) | Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy. | `list(string)` |
[
"/bin/sh",
"-c"
]
| no | | [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | [worker\_ami\_name\_filter](#input\_worker\_ami\_name\_filter) | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | | [worker\_ami\_name\_filter\_windows](#input\_worker\_ami\_name\_filter\_windows) | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | @@ -304,7 +298,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [cluster\_endpoint](#output\_cluster\_endpoint) | The endpoint for your EKS Kubernetes API. | | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster. | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster. | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready. | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster OIDC Issuer | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | The cluster primary security group ID created by the EKS cluster on 1.14 or later. Referred to as 'Cluster security group' in the EKS console. | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console. | @@ -314,8 +308,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [fargate\_iam\_role\_name](#output\_fargate\_iam\_role\_name) | IAM role name for EKS Fargate pods | | [fargate\_profile\_arns](#output\_fargate\_profile\_arns) | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | | [fargate\_profile\_ids](#output\_fargate\_profile\_ids) | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | -| [kubeconfig](#output\_kubeconfig) | kubectl config file contents for this EKS cluster. | -| [kubeconfig\_filename](#output\_kubeconfig\_filename) | The filename of the generated kubectl config. | +| [kubeconfig](#output\_kubeconfig) | kubectl config file contents for this EKS cluster. Will block on cluster creation until the cluster is really ready. | +| [kubeconfig\_filename](#output\_kubeconfig\_filename) | The filename of the generated kubectl config. Will block on cluster creation until the cluster is really ready. | | [node\_groups](#output\_node\_groups) | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true`. | | [security\_group\_rule\_cluster\_https\_worker\_ingress](#output\_security\_group\_rule\_cluster\_https\_worker\_ingress) | Security group rule responsible for allowing pods to communicate with the EKS cluster API. | diff --git a/aws_auth.tf b/aws_auth.tf index 91b0a23add..f549dcce77 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -64,7 +64,7 @@ locals { resource "kubernetes_config_map" "aws_auth" { count = var.create_eks && var.manage_aws_auth ? 1 : 0 - depends_on = [null_resource.wait_for_cluster[0]] + depends_on = [data.http.wait_for_cluster[0]] metadata { name = "aws-auth" @@ -72,7 +72,7 @@ resource "kubernetes_config_map" "aws_auth" { labels = merge( { "app.kubernetes.io/managed-by" = "Terraform" - # / are replaced by . because label validator fails in this lib + # / are replaced by . because label validator fails in this lib # https://github.com/kubernetes/apimachinery/blob/1bdd76d09076d4dc0362456e59c8f551f5f24a72/pkg/util/validation/validation.go#L166 "terraform.io/module" = "terraform-aws-modules.eks.aws" }, diff --git a/cluster.tf b/cluster.tf index 4306db3073..3034d950a5 100644 --- a/cluster.tf +++ b/cluster.tf @@ -64,21 +64,10 @@ resource "aws_security_group_rule" "cluster_private_access" { } -resource "null_resource" "wait_for_cluster" { - count = var.create_eks && var.manage_aws_auth ? 1 : 0 - - depends_on = [ - aws_eks_cluster.this, - aws_security_group_rule.cluster_private_access, - ] - - provisioner "local-exec" { - command = var.wait_for_cluster_cmd - interpreter = var.wait_for_cluster_interpreter - environment = { - ENDPOINT = aws_eks_cluster.this[0].endpoint - } - } +data "http" "wait_for_cluster" { + count = var.create_eks && var.manage_aws_auth ? 1 : 0 + url = format("%s/healthz", aws_eks_cluster.this[0].endpoint) + ca_certificate = base64decode(coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0]) } resource "aws_security_group" "cluster" { diff --git a/docs/faq.md b/docs/faq.md index 9fba65b7fa..3b9e118f6b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -107,7 +107,7 @@ You do not need to do anything extra since v12.1.0 of the module as long as the - `manage_aws_auth = true` on the module (default) - the kubernetes provider is correctly configured like in the [Usage Example](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/README.md#usage-example). Primarily the module's `cluster_id` output is used as input to the `aws_eks_cluster*` data sources. -The `cluster_id` depends on a `null_resource` that polls the EKS cluster's endpoint until it is alive. This blocks initialisation of the kubernetes provider. +The `cluster_id` depends on a `data.http.wait_for_cluster` that polls the EKS cluster's endpoint until it is alive. This blocks initialisation of the kubernetes provider. ## `aws_auth.tf: At 2:14: Unknown token: 2:14 IDENT` @@ -170,18 +170,6 @@ worker_groups = [ 4. With `kubectl get nodes` you can see cluster with mixed (Linux/Windows) nodes support. -## Deploying from Windows: `/bin/sh` file does not exist - -The module is almost pure Terraform apart from the `wait_for_cluster` `null_resource` that runs a local provisioner. The module has a default configuration for Unix-like systems. In order to run the provisioner on Windows systems you must set the interpreter to a valid value. [PR #795 (comment)](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/795#issuecomment-599191029) suggests the following value: -```hcl -module "eks" { - # ... - wait_for_cluster_interpreter = ["c:/git/bin/sh.exe", "-c"] -} -``` - -Alternatively, you can disable the `null_resource` by disabling creation of the `aws-auth` ConfigMap via setting `manage_aws_auth = false` on the module. The ConfigMap will then need creating via a different method. - ## Worker nodes with labels do not join a 1.16+ cluster Kubelet restricts the allowed list of labels in the `kubernetes.io` namespace that can be applied to nodes starting in 1.16. diff --git a/outputs.tf b/outputs.tf index 62d75aaf0f..147f036014 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,9 +1,10 @@ output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready." value = element(concat(aws_eks_cluster.this.*.id, [""]), 0) - # So that calling plans wait for the cluster to be available before attempting - # to use it. They will not need to duplicate this null_resource - depends_on = [null_resource.wait_for_cluster] + + # So that calling plans wait for the cluster to be available before attempting to use it. + # There is no need to duplicate this datasource + depends_on = [data.http.wait_for_cluster] } output "cluster_arn" { @@ -67,13 +68,21 @@ output "cloudwatch_log_group_arn" { } output "kubeconfig" { - description = "kubectl config file contents for this EKS cluster." + description = "kubectl config file contents for this EKS cluster. Will block on cluster creation until the cluster is really ready." value = local.kubeconfig + + # So that calling plans wait for the cluster to be available before attempting to use it. + # There is no need to duplicate this datasource + depends_on = [data.http.wait_for_cluster] } output "kubeconfig_filename" { - description = "The filename of the generated kubectl config." + description = "The filename of the generated kubectl config. Will block on cluster creation until the cluster is really ready." value = concat(local_file.kubeconfig.*.filename, [""])[0] + + # So that calling plans wait for the cluster to be available before attempting to use it. + # There is no need to duplicate this datasource + depends_on = [data.http.wait_for_cluster] } output "oidc_provider_arn" { diff --git a/variables.tf b/variables.tf index 6f904b6354..a48093d577 100644 --- a/variables.tf +++ b/variables.tf @@ -205,18 +205,6 @@ variable "cluster_delete_timeout" { default = "15m" } -variable "wait_for_cluster_cmd" { - description = "Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT" - type = string - default = "for i in `seq 1 60`; do if `command -v wget > /dev/null`; then wget --no-check-certificate -O - -q $ENDPOINT/healthz >/dev/null && exit 0 || true; else curl -k -s $ENDPOINT/healthz >/dev/null && exit 0 || true;fi; sleep 5; done; echo TIMEOUT && exit 1" -} - -variable "wait_for_cluster_interpreter" { - description = "Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy." - type = list(string) - default = ["/bin/sh", "-c"] -} - variable "cluster_create_security_group" { description = "Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`." type = bool diff --git a/versions.tf b/versions.tf index 6d70b06380..d6801b048e 100644 --- a/versions.tf +++ b/versions.tf @@ -4,9 +4,12 @@ terraform { required_providers { aws = ">= 3.37.0" local = ">= 1.4" - null = ">= 2.1" template = ">= 2.1" random = ">= 2.1" kubernetes = ">= 1.11.1" + http = { + source = "terraform-aws-modules/http" + version = ">= 2.2.0" + } } } From f7d09a2901aa83e962e7462634a08c5a8f67d7b7 Mon Sep 17 00:00:00 2001 From: Dov Salomon Date: Mon, 17 May 2021 04:25:38 -0400 Subject: [PATCH 294/962] feat: Remove dependency on deprecated `hashicorp/template` provider (#1297) BREAKING CHANGES: We remove the dependency on the deprecated `hashicorp/template` provider and use the Terraform built in `templatefile` function. This will broke some workflows due to previously being able to pass in the raw contents of a template file for processing. The `templatefile` function requires a template file that exists before running a plan. --- data.tf | 92 ------------------- examples/basic/versions.tf | 1 - examples/fargate/versions.tf | 1 - examples/irsa/versions.tf | 1 - examples/launch_templates/versions.tf | 1 - .../versions.tf | 1 - examples/managed_node_groups/versions.tf | 1 - examples/secrets_encryption/versions.tf | 1 - examples/spot_instances/versions.tf | 1 - local.tf | 88 ++++++++++++++++++ outputs.tf | 4 +- versions.tf | 1 - workers.tf | 2 +- workers_launch_template.tf | 2 +- 14 files changed, 92 insertions(+), 105 deletions(-) diff --git a/data.tf b/data.tf index 5909936031..a574e83bc6 100644 --- a/data.tf +++ b/data.tf @@ -55,98 +55,6 @@ data "aws_iam_policy_document" "cluster_assume_role_policy" { } } -data "template_file" "userdata" { - count = var.create_eks ? local.worker_group_count : 0 - template = lookup( - var.worker_groups[count.index], - "userdata_template_file", - file( - lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" - ? "${path.module}/templates/userdata_windows.tpl" - : "${path.module}/templates/userdata.sh.tpl" - ) - ) - - vars = merge({ - platform = lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] - endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] - cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] - pre_userdata = lookup( - var.worker_groups[count.index], - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - var.worker_groups[count.index], - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - var.worker_groups[count.index], - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - var.worker_groups[count.index], - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - var.worker_groups[count.index], - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) -} - -data "template_file" "launch_template_userdata" { - count = var.create_eks ? local.worker_group_launch_template_count : 0 - template = lookup( - var.worker_groups_launch_template[count.index], - "userdata_template_file", - file( - lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" - ? "${path.module}/templates/userdata_windows.tpl" - : "${path.module}/templates/userdata.sh.tpl" - ) - ) - - vars = merge({ - platform = lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] - endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] - cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] - pre_userdata = lookup( - var.worker_groups_launch_template[count.index], - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - var.worker_groups_launch_template[count.index], - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - var.worker_groups_launch_template[count.index], - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - var.worker_groups_launch_template[count.index], - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - var.worker_groups_launch_template[count.index], - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) -} - data "aws_iam_role" "custom_cluster_iam_role" { count = var.manage_cluster_iam_resources ? 0 : 1 name = var.cluster_iam_role_name diff --git a/examples/basic/versions.tf b/examples/basic/versions.tf index c9f4a4d350..c6f56daafa 100644 --- a/examples/basic/versions.tf +++ b/examples/basic/versions.tf @@ -5,7 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" null = ">= 2.1" - template = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/fargate/versions.tf b/examples/fargate/versions.tf index c9f4a4d350..c6f56daafa 100644 --- a/examples/fargate/versions.tf +++ b/examples/fargate/versions.tf @@ -5,7 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" null = ">= 2.1" - template = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/irsa/versions.tf b/examples/irsa/versions.tf index c9f4a4d350..c6f56daafa 100644 --- a/examples/irsa/versions.tf +++ b/examples/irsa/versions.tf @@ -5,7 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" null = ">= 2.1" - template = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/launch_templates/versions.tf b/examples/launch_templates/versions.tf index c9f4a4d350..c6f56daafa 100644 --- a/examples/launch_templates/versions.tf +++ b/examples/launch_templates/versions.tf @@ -5,7 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" null = ">= 2.1" - template = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/launch_templates_with_managed_node_groups/versions.tf b/examples/launch_templates_with_managed_node_groups/versions.tf index c9f4a4d350..c6f56daafa 100644 --- a/examples/launch_templates_with_managed_node_groups/versions.tf +++ b/examples/launch_templates_with_managed_node_groups/versions.tf @@ -5,7 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" null = ">= 2.1" - template = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/managed_node_groups/versions.tf b/examples/managed_node_groups/versions.tf index c9f4a4d350..c6f56daafa 100644 --- a/examples/managed_node_groups/versions.tf +++ b/examples/managed_node_groups/versions.tf @@ -5,7 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" null = ">= 2.1" - template = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/secrets_encryption/versions.tf b/examples/secrets_encryption/versions.tf index c9f4a4d350..c6f56daafa 100644 --- a/examples/secrets_encryption/versions.tf +++ b/examples/secrets_encryption/versions.tf @@ -5,7 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" null = ">= 2.1" - template = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/spot_instances/versions.tf b/examples/spot_instances/versions.tf index c9f4a4d350..c6f56daafa 100644 --- a/examples/spot_instances/versions.tf +++ b/examples/spot_instances/versions.tf @@ -5,7 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" null = ">= 2.1" - template = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/local.tf b/local.tf index 924f9d79e0..3367ae0571 100644 --- a/local.tf +++ b/local.tf @@ -150,4 +150,92 @@ locals { aws_authenticator_additional_args = var.kubeconfig_aws_authenticator_additional_args aws_authenticator_env_variables = var.kubeconfig_aws_authenticator_env_variables }) : "" + + userdata_rendered = [ + for index in range(var.create_eks ? local.worker_group_count : 0) : templatefile( + lookup( + var.worker_groups[index], + "userdata_template_file", + lookup(var.worker_groups[index], "platform", local.workers_group_defaults["platform"]) == "windows" + ? "${path.module}/templates/userdata_windows.tpl" + : "${path.module}/templates/userdata.sh.tpl" + ), + merge({ + platform = lookup(var.worker_groups[index], "platform", local.workers_group_defaults["platform"]) + cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] + cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] + pre_userdata = lookup( + var.worker_groups[index], + "pre_userdata", + local.workers_group_defaults["pre_userdata"], + ) + additional_userdata = lookup( + var.worker_groups[index], + "additional_userdata", + local.workers_group_defaults["additional_userdata"], + ) + bootstrap_extra_args = lookup( + var.worker_groups[index], + "bootstrap_extra_args", + local.workers_group_defaults["bootstrap_extra_args"], + ) + kubelet_extra_args = lookup( + var.worker_groups[index], + "kubelet_extra_args", + local.workers_group_defaults["kubelet_extra_args"], + ) + }, + lookup( + var.worker_groups[index], + "userdata_template_extra_args", + local.workers_group_defaults["userdata_template_extra_args"] + ) + ) + ) + ] + + launch_template_userdata_rendered = [ + for index in range(var.create_eks ? local.worker_group_launch_template_count : 0) : templatefile( + lookup( + var.worker_groups_launch_template[index], + "userdata_template_file", + lookup(var.worker_groups_launch_template[index], "platform", local.workers_group_defaults["platform"]) == "windows" + ? "${path.module}/templates/userdata_windows.tpl" + : "${path.module}/templates/userdata.sh.tpl" + ), + merge({ + platform = lookup(var.worker_groups_launch_template[index], "platform", local.workers_group_defaults["platform"]) + cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] + cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] + pre_userdata = lookup( + var.worker_groups_launch_template[index], + "pre_userdata", + local.workers_group_defaults["pre_userdata"], + ) + additional_userdata = lookup( + var.worker_groups_launch_template[index], + "additional_userdata", + local.workers_group_defaults["additional_userdata"], + ) + bootstrap_extra_args = lookup( + var.worker_groups_launch_template[index], + "bootstrap_extra_args", + local.workers_group_defaults["bootstrap_extra_args"], + ) + kubelet_extra_args = lookup( + var.worker_groups_launch_template[index], + "kubelet_extra_args", + local.workers_group_defaults["kubelet_extra_args"], + ) + }, + lookup( + var.worker_groups_launch_template[index], + "userdata_template_extra_args", + local.workers_group_defaults["userdata_template_extra_args"] + ) + ) + ) + ] } diff --git a/outputs.tf b/outputs.tf index 147f036014..b751fb59b1 100644 --- a/outputs.tf +++ b/outputs.tf @@ -109,8 +109,8 @@ output "workers_asg_names" { output "workers_user_data" { description = "User data of worker groups" value = concat( - data.template_file.userdata.*.rendered, - data.template_file.launch_template_userdata.*.rendered, + local.userdata_rendered, + local.launch_template_userdata_rendered, ) } diff --git a/versions.tf b/versions.tf index d6801b048e..57695c1282 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,6 @@ terraform { required_providers { aws = ">= 3.37.0" local = ">= 1.4" - template = ">= 2.1" random = ">= 2.1" kubernetes = ">= 1.11.1" http = { diff --git a/workers.tf b/workers.tf index 2bd75a27ff..0801c88ef2 100644 --- a/workers.tf +++ b/workers.tf @@ -205,7 +205,7 @@ resource "aws_launch_configuration" "workers" { "key_name", local.workers_group_defaults["key_name"], ) - user_data_base64 = base64encode(data.template_file.userdata.*.rendered[count.index]) + user_data_base64 = base64encode(local.userdata_rendered[count.index]) ebs_optimized = lookup( var.worker_groups[count.index], "ebs_optimized", diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 1602437e55..53e1b0928e 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -318,7 +318,7 @@ resource "aws_launch_template" "workers_launch_template" { local.workers_group_defaults["key_name"], ) user_data = base64encode( - data.template_file.launch_template_userdata.*.rendered[count.index], + local.launch_template_userdata_rendered[count.index], ) ebs_optimized = lookup( From 5cbbe803023637510fb295f17dbe3c0518c9500b Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 17 May 2021 11:06:19 +0200 Subject: [PATCH 295/962] fix: Remove provider from required providers (#1357) --- README.md | 4 ---- examples/basic/versions.tf | 1 - examples/fargate/versions.tf | 1 - examples/irsa/versions.tf | 1 - examples/launch_templates/versions.tf | 1 - .../launch_templates_with_managed_node_groups/versions.tf | 1 - examples/managed_node_groups/versions.tf | 1 - examples/secrets_encryption/versions.tf | 1 - examples/spot_instances/versions.tf | 1 - 9 files changed, 12 deletions(-) diff --git a/README.md b/README.md index 87b7ab085c..75ba866300 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | -| [template](#requirement\_template) | >= 2.1 | ## Providers @@ -157,7 +156,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | | [random](#provider\_random) | >= 2.1 | -| [template](#provider\_template) | >= 2.1 | ## Modules @@ -217,8 +215,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [aws_iam_role.custom_cluster_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [http_http.wait_for_cluster](https://registry.terraform.io/providers/terraform-aws-modules/http/latest/docs/data-sources/http) | data source | -| [template_file.launch_template_userdata](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | -| [template_file.userdata](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | ## Inputs diff --git a/examples/basic/versions.tf b/examples/basic/versions.tf index c6f56daafa..6e29ae8f1b 100644 --- a/examples/basic/versions.tf +++ b/examples/basic/versions.tf @@ -4,7 +4,6 @@ terraform { required_providers { aws = ">= 3.22.0" local = ">= 1.4" - null = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/fargate/versions.tf b/examples/fargate/versions.tf index c6f56daafa..6e29ae8f1b 100644 --- a/examples/fargate/versions.tf +++ b/examples/fargate/versions.tf @@ -4,7 +4,6 @@ terraform { required_providers { aws = ">= 3.22.0" local = ">= 1.4" - null = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/irsa/versions.tf b/examples/irsa/versions.tf index c6f56daafa..6e29ae8f1b 100644 --- a/examples/irsa/versions.tf +++ b/examples/irsa/versions.tf @@ -4,7 +4,6 @@ terraform { required_providers { aws = ">= 3.22.0" local = ">= 1.4" - null = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/launch_templates/versions.tf b/examples/launch_templates/versions.tf index c6f56daafa..6e29ae8f1b 100644 --- a/examples/launch_templates/versions.tf +++ b/examples/launch_templates/versions.tf @@ -4,7 +4,6 @@ terraform { required_providers { aws = ">= 3.22.0" local = ">= 1.4" - null = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/launch_templates_with_managed_node_groups/versions.tf b/examples/launch_templates_with_managed_node_groups/versions.tf index c6f56daafa..6e29ae8f1b 100644 --- a/examples/launch_templates_with_managed_node_groups/versions.tf +++ b/examples/launch_templates_with_managed_node_groups/versions.tf @@ -4,7 +4,6 @@ terraform { required_providers { aws = ">= 3.22.0" local = ">= 1.4" - null = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/managed_node_groups/versions.tf b/examples/managed_node_groups/versions.tf index c6f56daafa..6e29ae8f1b 100644 --- a/examples/managed_node_groups/versions.tf +++ b/examples/managed_node_groups/versions.tf @@ -4,7 +4,6 @@ terraform { required_providers { aws = ">= 3.22.0" local = ">= 1.4" - null = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/secrets_encryption/versions.tf b/examples/secrets_encryption/versions.tf index c6f56daafa..6e29ae8f1b 100644 --- a/examples/secrets_encryption/versions.tf +++ b/examples/secrets_encryption/versions.tf @@ -4,7 +4,6 @@ terraform { required_providers { aws = ">= 3.22.0" local = ">= 1.4" - null = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } diff --git a/examples/spot_instances/versions.tf b/examples/spot_instances/versions.tf index c6f56daafa..6e29ae8f1b 100644 --- a/examples/spot_instances/versions.tf +++ b/examples/spot_instances/versions.tf @@ -4,7 +4,6 @@ terraform { required_providers { aws = ">= 3.22.0" local = ">= 1.4" - null = ">= 2.1" random = ">= 2.1" kubernetes = "~> 1.11" } From 19e6be9df338396ca3dfeebb2994c9a91b54f465 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 17 May 2021 14:10:33 +0200 Subject: [PATCH 296/962] feat: Add timeout of 5mn when waiting for cluster (#1359) --- README.md | 4 ++-- cluster.tf | 1 + versions.tf | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 75ba866300..d1b01cbb9e 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.37.0 | -| [http](#requirement\_http) | >= 2.2.0 | +| [http](#requirement\_http) | >= 2.3.0 | | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -152,7 +152,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.37.0 | -| [http](#provider\_http) | >= 2.2.0 | +| [http](#provider\_http) | >= 2.3.0 | | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | | [random](#provider\_random) | >= 2.1 | diff --git a/cluster.tf b/cluster.tf index 3034d950a5..7f7a726946 100644 --- a/cluster.tf +++ b/cluster.tf @@ -68,6 +68,7 @@ data "http" "wait_for_cluster" { count = var.create_eks && var.manage_aws_auth ? 1 : 0 url = format("%s/healthz", aws_eks_cluster.this[0].endpoint) ca_certificate = base64decode(coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0]) + timeout = 300 } resource "aws_security_group" "cluster" { diff --git a/versions.tf b/versions.tf index 57695c1282..57d35d6329 100644 --- a/versions.tf +++ b/versions.tf @@ -8,7 +8,7 @@ terraform { kubernetes = ">= 1.11.1" http = { source = "terraform-aws-modules/http" - version = ">= 2.2.0" + version = ">= 2.3.0" } } } From 32f70afe602cc96e89f31e3562bf237d80482a4b Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 17 May 2021 16:20:35 +0200 Subject: [PATCH 297/962] feat: Drop `asg_recreate_on_change` feature to encourage the usage of Instance Refresh for EC2 Auto Scaling (#1360) BREAKING CHANGES: This module used `random_pet` resources to create a random name for the autoscaling group to force the autoscaling group to be re-created when the launch configuration or launch template was changed (if `recreate_asg_when_lc_changes = true` was set), causing the instances to be removed and re-provisioned each time there was an update. Those random_pet resources has been removed and in its place there is now a set of functionality provided by AWS and the Terraform AWS provider - Instance Refresh. We encourage those users to move on Instance Refresh for EC2 Auto Scaling. --- local.tf | 1 - workers.tf | 18 +----------------- workers_launch_template.tf | 26 +------------------------- 3 files changed, 2 insertions(+), 43 deletions(-) diff --git a/local.tf b/local.tf index 3367ae0571..44918625fb 100644 --- a/local.tf +++ b/local.tf @@ -35,7 +35,6 @@ locals { asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. asg_force_delete = false # Enable forced deletion for the autoscaling group. asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. - asg_recreate_on_change = false # Recreate the autoscaling group when the Launch Template or Launch Configuration change. default_cooldown = null # The amount of time, in seconds, after a scaling activity completes before another scaling activity can start. health_check_type = null # Controls how health checking is done. Valid values are "EC2" or "ELB". health_check_grace_period = null # Time in seconds after instance comes into service before checking health. diff --git a/workers.tf b/workers.tf index 0801c88ef2..ae5a71113a 100644 --- a/workers.tf +++ b/workers.tf @@ -7,8 +7,7 @@ resource "aws_autoscaling_group" "workers" { compact( [ coalescelist(aws_eks_cluster.this[*].name, [""])[0], - lookup(var.worker_groups[count.index], "name", count.index), - lookup(var.worker_groups[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers[count.index].id : "" + lookup(var.worker_groups[count.index], "name", count.index) ] ) ) @@ -326,21 +325,6 @@ resource "aws_launch_configuration" "workers" { ] } -resource "random_pet" "workers" { - count = var.create_eks ? local.worker_group_count : 0 - - separator = "-" - length = 2 - - keepers = { - lc_name = aws_launch_configuration.workers[count.index].name - } - - lifecycle { - create_before_destroy = true - } -} - resource "aws_security_group" "workers" { count = var.worker_create_security_group && var.create_eks ? 1 : 0 name_prefix = var.cluster_name diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 53e1b0928e..14564c1708 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -7,8 +7,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { compact( [ coalescelist(aws_eks_cluster.this[*].name, [""])[0], - lookup(var.worker_groups_launch_template[count.index], "name", count.index), - lookup(var.worker_groups_launch_template[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers_launch_template[count.index].id : "" + lookup(var.worker_groups_launch_template[count.index], "name", count.index) ] ) ) @@ -531,29 +530,6 @@ resource "aws_launch_template" "workers_launch_template" { ] } -resource "random_pet" "workers_launch_template" { - count = var.create_eks ? local.worker_group_launch_template_count : 0 - - separator = "-" - length = 2 - - keepers = { - lt_name = join( - "-", - compact( - [ - aws_launch_template.workers_launch_template[count.index].name, - aws_launch_template.workers_launch_template[count.index].latest_version - ] - ) - ) - } - - lifecycle { - create_before_destroy = true - } -} - resource "aws_iam_instance_profile" "workers_launch_template" { count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_launch_template_count : 0 name_prefix = coalescelist(aws_eks_cluster.this[*].name, [""])[0] From 68e9df92c3f1e41114dfa24bf914a98228a99323 Mon Sep 17 00:00:00 2001 From: Benjamin Ash <22301927+bashims@users.noreply.github.com> Date: Mon, 17 May 2021 16:12:53 -0400 Subject: [PATCH 298/962] feat: Add support for Auto Scaling Group Instance Refresh for self-managed worker groups (#1224) Co-authored-by: Thierno IB. BARRY --- .gitignore | 1 + README.md | 5 +- examples/instance_refresh/main.tf | 234 +++++++++++++++++++++++++ examples/instance_refresh/outputs.tf | 34 ++++ examples/instance_refresh/variables.tf | 18 ++ examples/instance_refresh/versions.tf | 11 ++ local.tf | 7 +- workers.tf | 27 +++ workers_launch_template.tf | 28 ++- 9 files changed, 360 insertions(+), 5 deletions(-) create mode 100644 examples/instance_refresh/main.tf create mode 100644 examples/instance_refresh/outputs.tf create mode 100644 examples/instance_refresh/variables.tf create mode 100644 examples/instance_refresh/versions.tf diff --git a/.gitignore b/.gitignore index 4759a205e3..c78b7d1e2c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ eks-admin-cluster-role-binding.yaml eks-admin-service-account.yaml config-map-aws-auth*.yaml kubeconfig_* +.idea ################################################################# # Default .gitignore content for all terraform-aws-modules below diff --git a/README.md b/README.md index d1b01cbb9e..6169d70799 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ An example of harming update was the removal of several commonly used, but depre By default, this module manages the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready. We implemented a "retry" logic with a fork of the http provider https://github.com/terraform-aws-modules/terraform-provider-http. This fork adds the support of a self-signed CA certificate. The original PR can be found at https://github.com/hashicorp/terraform-provider-http/pull/29. +Setting `instance_refresh_enabled` to true will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. Find the complete example here [instance_refresh](examples/instance_refresh). + ## Usage example A full example leveraging other community modules is contained in the [examples/basic directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). @@ -155,7 +157,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [http](#provider\_http) | >= 2.3.0 | | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | -| [random](#provider\_random) | >= 2.1 | ## Modules @@ -202,8 +203,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [aws_security_group_rule.workers_ingress_self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | | [local_file.kubeconfig](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [random_pet.workers](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | -| [random_pet.workers_launch_template](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | | [aws_ami.eks_worker](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_ami.eks_worker_windows](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | diff --git a/examples/instance_refresh/main.tf b/examples/instance_refresh/main.tf new file mode 100644 index 0000000000..137df25b4a --- /dev/null +++ b/examples/instance_refresh/main.tf @@ -0,0 +1,234 @@ +provider "aws" { + region = var.region +} + +data "aws_caller_identity" "current" {} + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + } +} + +data "aws_availability_zones" "available" { +} + +locals { + cluster_name = "test-refresh-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0.0" + + name = local.cluster_name + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_dns_hostnames = true +} + +data "aws_iam_policy_document" "node_term" { + statement { + effect = "Allow" + actions = [ + "ec2:DescribeInstances", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeTags", + ] + resources = [ + "*", + ] + } + statement { + effect = "Allow" + actions = [ + "autoscaling:CompleteLifecycleAction", + ] + resources = module.eks.workers_asg_arns + } + statement { + effect = "Allow" + actions = [ + "sqs:DeleteMessage", + "sqs:ReceiveMessage" + ] + resources = [ + module.node_term_sqs.sqs_queue_arn + ] + } +} + +resource "aws_iam_policy" "node_term" { + name = "node-term-${local.cluster_name}" + policy = data.aws_iam_policy_document.node_term.json +} + +resource "aws_iam_role_policy_attachment" "node_term_policy" { + policy_arn = aws_iam_policy.node_term.arn + role = module.eks.worker_iam_role_name +} + +data "aws_iam_policy_document" "node_term_events" { + statement { + effect = "Allow" + principals { + type = "Service" + identifiers = [ + "events.amazonaws.com", + "sqs.amazonaws.com", + ] + } + actions = [ + "sqs:SendMessage", + ] + resources = [ + "arn:aws:sqs:${var.region}:${data.aws_caller_identity.current.account_id}:${local.cluster_name}", + ] + } +} + +module "node_term_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "~> 3.0.0" + name = local.cluster_name + message_retention_seconds = 300 + policy = data.aws_iam_policy_document.node_term_events.json +} + +resource "aws_cloudwatch_event_rule" "node_term_event_rule" { + name = "${local.cluster_name}-nth-rule" + description = "Node termination event rule" + event_pattern = jsonencode( + { + "source" : [ + "aws.autoscaling" + ], + "detail-type" : [ + "EC2 Instance-terminate Lifecycle Action" + ] + "resources" : module.eks.workers_asg_arns + } + ) +} + +resource "aws_cloudwatch_event_target" "node_term_event_target" { + rule = aws_cloudwatch_event_rule.node_term_event_rule.name + target_id = "ANTHandler" + arn = module.node_term_sqs.sqs_queue_arn +} + +module "node_term_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "4.1.0" + create_role = true + role_description = "IRSA role for ANTH, cluster ${local.cluster_name}" + role_name_prefix = local.cluster_name + provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") + role_policy_arns = [aws_iam_policy.node_term.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:${var.namespace}:${var.serviceaccount}"] +} + +resource "helm_release" "anth" { + depends_on = [ + module.eks + ] + + name = "aws-node-termination-handler" + namespace = var.namespace + repository = "https://aws.github.io/eks-charts" + chart = "aws-node-termination-handler" + version = var.aws_node_termination_handler_chart_version + create_namespace = true + + set { + name = "awsRegion" + value = var.region + } + set { + name = "serviceAccount.name" + value = var.serviceaccount + } + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.node_term_role.iam_role_arn + type = "string" + } + set { + name = "enableSqsTerminationDraining" + value = "true" + } + set { + name = "queueURL" + value = module.node_term_sqs.sqs_queue_id + } + set { + name = "logLevel" + value = "DEBUG" + } +} + +# Creating the lifecycle-hook outside of the ASG resource's `initial_lifecycle_hook` +# ensures that node termination does not require the lifecycle action to be completed, +# and thus allows the ASG to be destroyed cleanly. +resource "aws_autoscaling_lifecycle_hook" "node_term" { + name = "node_term-${local.cluster_name}" + autoscaling_group_name = module.eks.workers_asg_names[0] + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + heartbeat_timeout = 300 + default_result = "CONTINUE" +} + +module "eks" { + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.19" + subnets = module.vpc.public_subnets + vpc_id = module.vpc.vpc_id + enable_irsa = true + worker_groups_launch_template = [ + { + name = "refresh" + asg_max_size = 2 + asg_desired_capacity = 2 + instance_refresh_enabled = true + instance_refresh_triggers = ["tag"] + public_ip = true + metadata_http_put_response_hop_limit = 3 + tags = [ + { + key = "aws-node-termination-handler/managed" + value = "" + propagate_at_launch = true + }, + { + key = "foo" + value = "buzz" + propagate_at_launch = true + }, + ] + }, + ] +} diff --git a/examples/instance_refresh/outputs.tf b/examples/instance_refresh/outputs.tf new file mode 100644 index 0000000000..6b767427fb --- /dev/null +++ b/examples/instance_refresh/outputs.tf @@ -0,0 +1,34 @@ +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = module.eks.cluster_security_group_id +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = module.eks.kubeconfig +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = module.eks.config_map_aws_auth +} + +output "region" { + description = "AWS region." + value = var.region +} + +output "sqs_queue_asg_notification_arn" { + description = "SQS queue ASG notification ARN" + value = module.node_term_sqs.sqs_queue_arn +} + +output "sqs_queue_asg_notification_url" { + description = "SQS queue ASG notification URL" + value = module.node_term_sqs.sqs_queue_id +} diff --git a/examples/instance_refresh/variables.tf b/examples/instance_refresh/variables.tf new file mode 100644 index 0000000000..96fc26df92 --- /dev/null +++ b/examples/instance_refresh/variables.tf @@ -0,0 +1,18 @@ +variable "region" { + default = "us-west-2" +} + +variable "aws_node_termination_handler_chart_version" { + description = "Version of the aws-node-termination-handler Helm chart to install." + default = "0.15.0" +} + +variable "namespace" { + description = "Namespace for the aws-node-termination-handler." + default = "kube-system" +} + +variable "serviceaccount" { + description = "Serviceaccount for the aws-node-termination-handler." + default = "aws-node-termination-handler" +} diff --git a/examples/instance_refresh/versions.tf b/examples/instance_refresh/versions.tf new file mode 100644 index 0000000000..67281c8d51 --- /dev/null +++ b/examples/instance_refresh/versions.tf @@ -0,0 +1,11 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = ">= 3.22.0" + local = ">= 1.4" + random = ">= 2.1" + kubernetes = "~> 1.11" + helm = "~> 2.1.2" + } +} diff --git a/local.tf b/local.tf index 44918625fb..3fa7c1f442 100644 --- a/local.tf +++ b/local.tf @@ -34,7 +34,7 @@ locals { asg_max_size = "3" # Maximum worker capacity in the autoscaling group. asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. asg_force_delete = false # Enable forced deletion for the autoscaling group. - asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. + asg_initial_lifecycle_hooks = [] # Initial lifecycle hook for the autoscaling group. default_cooldown = null # The amount of time, in seconds, after a scaling activity completes before another scaling activity can start. health_check_type = null # Controls how health checking is done. Valid values are "EC2" or "ELB". health_check_grace_period = null # Time in seconds after instance comes into service before checking health. @@ -95,6 +95,11 @@ locals { spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price max_instance_lifetime = 0 # Maximum number of seconds instances can run in the ASG. 0 is unlimited. elastic_inference_accelerator = null # Type of elastic inference accelerator to be attached. Example values are eia1.medium, eia2.large, etc. + instance_refresh_enabled = false # Enable instance refresh for the worker autoscaling group. + instance_refresh_strategy = "Rolling" # Strategy to use for instance refresh. Default is 'Rolling' which the only valid value. + instance_refresh_min_healthy_percentage = 90 # The amount of capacity in the ASG that must remain healthy during an instance refresh, as a percentage of the ASG's desired capacity. + instance_refresh_instance_warmup = null # The number of seconds until a newly launched instance is configured and ready to use. Defaults to the ASG's health check grace period. + instance_refresh_triggers = [] # Set of additional property names that will trigger an Instance Refresh. A refresh will always be triggered by a change in any of launch_configuration, launch_template, or mixed_instances_policy. } workers_group_defaults = merge( diff --git a/workers.tf b/workers.tf index ae5a71113a..897805df98 100644 --- a/workers.tf +++ b/workers.tf @@ -162,6 +162,33 @@ resource "aws_autoscaling_group" "workers" { } } + # logic duplicated in workers_launch_template.tf + dynamic "instance_refresh" { + for_each = lookup(var.worker_groups[count.index], + "instance_refresh_enabled", + local.workers_group_defaults["instance_refresh_enabled"]) ? [1] : [] + content { + strategy = lookup( + var.worker_groups[count.index], "instance_refresh_strategy", + local.workers_group_defaults["instance_refresh_strategy"] + ) + preferences { + instance_warmup = lookup( + var.worker_groups[count.index], "instance_refresh_instance_warmup", + local.workers_group_defaults["instance_refresh_instance_warmup"] + ) + min_healthy_percentage = lookup( + var.worker_groups[count.index], "instance_refresh_min_healthy_percentage", + local.workers_group_defaults["instance_refresh_min_healthy_percentage"] + ) + } + triggers = lookup( + var.worker_groups[count.index], "instance_refresh_triggers", + local.workers_group_defaults["instance_refresh_triggers"] + ) + } + } + lifecycle { create_before_destroy = true ignore_changes = [desired_capacity] diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 14564c1708..f22a48f294 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -156,7 +156,6 @@ resource "aws_autoscaling_group" "workers_launch_template" { instance_type = override.value } } - } } } @@ -238,6 +237,33 @@ resource "aws_autoscaling_group" "workers_launch_template" { } } + # logic duplicated in workers.tf + dynamic "instance_refresh" { + for_each = lookup(var.worker_groups_launch_template[count.index], + "instance_refresh_enabled", + local.workers_group_defaults["instance_refresh_enabled"]) ? [1] : [] + content { + strategy = lookup( + var.worker_groups_launch_template[count.index], "instance_refresh_strategy", + local.workers_group_defaults["instance_refresh_strategy"] + ) + preferences { + instance_warmup = lookup( + var.worker_groups_launch_template[count.index], "instance_refresh_instance_warmup", + local.workers_group_defaults["instance_refresh_instance_warmup"] + ) + min_healthy_percentage = lookup( + var.worker_groups_launch_template[count.index], "instance_refresh_min_healthy_percentage", + local.workers_group_defaults["instance_refresh_min_healthy_percentage"] + ) + } + triggers = lookup( + var.worker_groups_launch_template[count.index], "instance_refresh_triggers", + local.workers_group_defaults["instance_refresh_triggers"] + ) + } + } + lifecycle { create_before_destroy = true ignore_changes = [desired_capacity] From acbf51c33342ce2b1d6b05f55ee9e149e7c26621 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 17 May 2021 22:23:43 +0200 Subject: [PATCH 299/962] chore: Update changelog (#1363) --- CHANGELOG.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b3c9f803a..3364d355a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,30 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v16.0.0] - 2021-05-17 +FEATURES: +- Add support for Auto Scaling Group Instance Refresh for self-managed worker groups ([#1224](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1224)) +- Drop `asg_recreate_on_change` feature to encourage the usage of Instance Refresh for EC2 Auto Scaling ([#1360](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1360)) +- Add timeout of 5mn when waiting for cluster ([#1359](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1359)) +- Remove dependency on deprecated `hashicorp/template` provider ([#1297](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1297)) +- Replace the local-exec script with a http datasource for waiting cluster ([#1339](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1339)) + +BUG FIXES: +- Remove provider from required providers ([#1357](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1357)) +- Bump AWS provider version to add Warm Pool support ([#1340](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1340)) + +CI: +- Bump terraform-docs to 0.13 ([#1335](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1335)) + +BREAKING CHANGES: +- This module used `random_pet` resources to create a random name for the autoscaling group to force the autoscaling group to be re-created when the launch configuration or launch template was changed (if `recreate_asg_when_lc_changes = true` was set), causing the instances to be removed and re-provisioned each time there was an update. Those random_pet resources has been removed and in its place there is now a set of functionality provided by AWS and the Terraform AWS provider - Instance Refresh. We encourage those users to move on Instance Refresh for EC2 Auto Scaling. +- We remove the dependency on the deprecated `hashicorp/template` provider and use the Terraform built in `templatefile` function. This will broke some workflows due to previously being able to pass in the raw contents of a template file for processing. The `templatefile` function requires a template file that exists before running a plan. + +NOTES: +- Using the [terraform-aws-modules/http](https://registry.terraform.io/providers/terraform-aws-modules/http/latest) provider is a more platform agnostic way to wait for the cluster availability than using a local-exec. With this change we're able to provision EKS clusters and manage the `aws_auth` configmap while still using the `hashicorp/tfc-agent` docker image. + + ## [v15.2.0] - 2021-05-04 FEATURES: @@ -277,7 +301,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.2.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.0...HEAD +[v16.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.2.0...v16.0.0 [v15.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.1.0...v15.2.0 [v15.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.0.0...v15.1.0 [v15.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v14.0.0...v15.0.0 From 91545abd75194183ad6dcaf57734c05e10817c32 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 19 May 2021 00:04:24 +0200 Subject: [PATCH 300/962] fix: Bump `terraform-aws-modules/http` provider version to support darwin arm64 release (#1369) --- README.md | 4 ++-- versions.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6169d70799..9b31790031 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.37.0 | -| [http](#requirement\_http) | >= 2.3.0 | +| [http](#requirement\_http) | >= 2.4.0 | | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -154,7 +154,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.37.0 | -| [http](#provider\_http) | >= 2.3.0 | +| [http](#provider\_http) | >= 2.4.0 | | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | diff --git a/versions.tf b/versions.tf index 57d35d6329..7fc51f80b0 100644 --- a/versions.tf +++ b/versions.tf @@ -8,7 +8,7 @@ terraform { kubernetes = ">= 1.11.1" http = { source = "terraform-aws-modules/http" - version = ">= 2.3.0" + version = ">= 2.4.0" } } } From 7c654a62b7ec1d057d3a7d3aa2d93010afa34db6 Mon Sep 17 00:00:00 2001 From: Steve Hipwell Date: Wed, 19 May 2021 11:40:54 +0100 Subject: [PATCH 301/962] docs: Use IRSA for Node Termination Handler IAM policy attachement in Instance Refresh example (#1373) --- examples/instance_refresh/main.tf | 85 ++++++++++++++++++---------- examples/instance_refresh/outputs.tf | 4 +- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/examples/instance_refresh/main.tf b/examples/instance_refresh/main.tf index 137df25b4a..e19b4b4e21 100644 --- a/examples/instance_refresh/main.tf +++ b/examples/instance_refresh/main.tf @@ -1,3 +1,5 @@ +# Based on the official aws-node-termination-handler setup guide at https://github.com/aws/aws-node-termination-handler#infrastructure-setup + provider "aws" { region = var.region } @@ -50,7 +52,7 @@ module "vpc" { enable_dns_hostnames = true } -data "aws_iam_policy_document" "node_term" { +data "aws_iam_policy_document" "aws_node_termination_handler" { statement { effect = "Allow" actions = [ @@ -76,22 +78,17 @@ data "aws_iam_policy_document" "node_term" { "sqs:ReceiveMessage" ] resources = [ - module.node_term_sqs.sqs_queue_arn + module.aws_node_termination_handler_sqs.sqs_queue_arn ] } } -resource "aws_iam_policy" "node_term" { - name = "node-term-${local.cluster_name}" - policy = data.aws_iam_policy_document.node_term.json -} - -resource "aws_iam_role_policy_attachment" "node_term_policy" { - policy_arn = aws_iam_policy.node_term.arn - role = module.eks.worker_iam_role_name +resource "aws_iam_policy" "aws_node_termination_handler" { + name = "${local.cluster_name}-aws-node-termination-handler" + policy = data.aws_iam_policy_document.aws_node_termination_handler.json } -data "aws_iam_policy_document" "node_term_events" { +data "aws_iam_policy_document" "aws_node_termination_handler_events" { statement { effect = "Allow" principals { @@ -110,16 +107,16 @@ data "aws_iam_policy_document" "node_term_events" { } } -module "node_term_sqs" { +module "aws_node_termination_handler_sqs" { source = "terraform-aws-modules/sqs/aws" version = "~> 3.0.0" name = local.cluster_name message_retention_seconds = 300 - policy = data.aws_iam_policy_document.node_term_events.json + policy = data.aws_iam_policy_document.aws_node_termination_handler_events.json } -resource "aws_cloudwatch_event_rule" "node_term_event_rule" { - name = "${local.cluster_name}-nth-rule" +resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_asg" { + name = "${local.cluster_name}-asg-termination" description = "Node termination event rule" event_pattern = jsonencode( { @@ -134,24 +131,46 @@ resource "aws_cloudwatch_event_rule" "node_term_event_rule" { ) } -resource "aws_cloudwatch_event_target" "node_term_event_target" { - rule = aws_cloudwatch_event_rule.node_term_event_rule.name - target_id = "ANTHandler" - arn = module.node_term_sqs.sqs_queue_arn +resource "aws_cloudwatch_event_target" "aws_node_termination_handler_asg" { + target_id = "${local.cluster_name}-asg-termination" + rule = aws_cloudwatch_event_rule.aws_node_termination_handler_asg.name + arn = module.aws_node_termination_handler_sqs.sqs_queue_arn } -module "node_term_role" { +resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_spot" { + name = "${local.cluster_name}-spot-termination" + description = "Node termination event rule" + event_pattern = jsonencode( + { + "source" : [ + "aws.ec2" + ], + "detail-type" : [ + "EC2 Spot Instance Interruption Warning" + ] + "resources" : module.eks.workers_asg_arns + } + ) +} + +resource "aws_cloudwatch_event_target" "aws_node_termination_handler_spot" { + target_id = "${local.cluster_name}-spot-termination" + rule = aws_cloudwatch_event_rule.aws_node_termination_handler_spot.name + arn = module.aws_node_termination_handler_sqs.sqs_queue_arn +} + +module "aws_node_termination_handler_role" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" version = "4.1.0" create_role = true role_description = "IRSA role for ANTH, cluster ${local.cluster_name}" role_name_prefix = local.cluster_name provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") - role_policy_arns = [aws_iam_policy.node_term.arn] + role_policy_arns = [aws_iam_policy.aws_node_termination_handler.arn] oidc_fully_qualified_subjects = ["system:serviceaccount:${var.namespace}:${var.serviceaccount}"] } -resource "helm_release" "anth" { +resource "helm_release" "aws_node_termination_handler" { depends_on = [ module.eks ] @@ -173,29 +192,35 @@ resource "helm_release" "anth" { } set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.node_term_role.iam_role_arn + value = module.aws_node_termination_handler_role.iam_role_arn type = "string" } set { name = "enableSqsTerminationDraining" value = "true" } + set { + name = "enableSpotInterruptionDraining" + value = "true" + } set { name = "queueURL" - value = module.node_term_sqs.sqs_queue_id + value = module.aws_node_termination_handler_sqs.sqs_queue_id } set { name = "logLevel" - value = "DEBUG" + value = "debug" } } # Creating the lifecycle-hook outside of the ASG resource's `initial_lifecycle_hook` # ensures that node termination does not require the lifecycle action to be completed, # and thus allows the ASG to be destroyed cleanly. -resource "aws_autoscaling_lifecycle_hook" "node_term" { - name = "node_term-${local.cluster_name}" - autoscaling_group_name = module.eks.workers_asg_names[0] +resource "aws_autoscaling_lifecycle_hook" "aws_node_termination_handler" { + for_each = toset(module.eks.workers_asg_names) + + name = "aws-node-termination-handler" + autoscaling_group_name = each.value lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" heartbeat_timeout = 300 default_result = "CONTINUE" @@ -227,8 +252,8 @@ module "eks" { key = "foo" value = "buzz" propagate_at_launch = true - }, + } ] - }, + } ] } diff --git a/examples/instance_refresh/outputs.tf b/examples/instance_refresh/outputs.tf index 6b767427fb..3b981f1f1b 100644 --- a/examples/instance_refresh/outputs.tf +++ b/examples/instance_refresh/outputs.tf @@ -25,10 +25,10 @@ output "region" { output "sqs_queue_asg_notification_arn" { description = "SQS queue ASG notification ARN" - value = module.node_term_sqs.sqs_queue_arn + value = module.aws_node_termination_handler_sqs.sqs_queue_arn } output "sqs_queue_asg_notification_url" { description = "SQS queue ASG notification URL" - value = module.node_term_sqs.sqs_queue_id + value = module.aws_node_termination_handler_sqs.sqs_queue_id } From 468a4b5faee0daac39d40d0d944987055c8267cb Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 19 May 2021 15:33:43 +0200 Subject: [PATCH 302/962] chore: update changelog (#1374) --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3364d355a8..f9b26f1394 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v16.0.1] - 2021-05-19 +BUG FIXES: +- Bump `terraform-aws-modules/http` provider version to support darwin arm64 release ([#1369](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1369)) + +DOCS: +- Use IRSA for Node Termination Handler IAM policy attachement in Instance Refresh example ([#1373](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1373)) + + ## [v16.0.0] - 2021-05-17 FEATURES: @@ -301,7 +310,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.1...HEAD +[v16.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.0...v16.0.1 [v16.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.2.0...v16.0.0 [v15.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.1.0...v15.2.0 [v15.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.0.0...v15.1.0 From a639cf5138004ee9d7be43648be65756ec1a3699 Mon Sep 17 00:00:00 2001 From: Tamas Millian Date: Wed, 19 May 2021 14:50:43 +0100 Subject: [PATCH 303/962] fix: Add description for private API ingress Security Group Rule (#1299) --- cluster.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster.tf b/cluster.tf index 7f7a726946..7acfd3367b 100644 --- a/cluster.tf +++ b/cluster.tf @@ -53,6 +53,7 @@ resource "aws_eks_cluster" "this" { } resource "aws_security_group_rule" "cluster_private_access" { + description = "Allow private K8S API ingress from custom source." count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access ? 1 : 0 type = "ingress" from_port = 443 From a2761ad685e5a0b6d0e882a9ead2a629febe284e Mon Sep 17 00:00:00 2001 From: Benjamin Ash <22301927+bashims@users.noreply.github.com> Date: Wed, 19 May 2021 10:34:59 -0400 Subject: [PATCH 304/962] fix: set an ASG's launch template version to an explicit version to automatically trigger instance refresh (#1370) NOTES: Set an ASG's launch template version to an explicit version automatically. This will ensure that an instance refresh will be triggered whenever the launch template changes. The default `launch_template_version` is now used to determine the latest or default version of the created launch template for self-managed worker groups. Signed-off-by: Benjamin Ash Co-authored-by: Thierno IB. BARRY --- examples/instance_refresh/main.tf | 9 +++++---- local.tf | 3 ++- workers_launch_template.tf | 22 ++++++++++++++++++++-- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/examples/instance_refresh/main.tf b/examples/instance_refresh/main.tf index e19b4b4e21..dd7af88a2e 100644 --- a/examples/instance_refresh/main.tf +++ b/examples/instance_refresh/main.tf @@ -217,10 +217,9 @@ resource "helm_release" "aws_node_termination_handler" { # ensures that node termination does not require the lifecycle action to be completed, # and thus allows the ASG to be destroyed cleanly. resource "aws_autoscaling_lifecycle_hook" "aws_node_termination_handler" { - for_each = toset(module.eks.workers_asg_names) - + count = length(module.eks.workers_asg_names) name = "aws-node-termination-handler" - autoscaling_group_name = each.value + autoscaling_group_name = module.eks.workers_asg_names[count.index] lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" heartbeat_timeout = 300 default_result = "CONTINUE" @@ -239,9 +238,11 @@ module "eks" { asg_max_size = 2 asg_desired_capacity = 2 instance_refresh_enabled = true - instance_refresh_triggers = ["tag"] + instance_refresh_instance_warmup = 60 public_ip = true metadata_http_put_response_hop_limit = 3 + update_default_version = true + instance_refresh_triggers = ["tag"] tags = [ { key = "aws-node-termination-handler/managed" diff --git a/local.tf b/local.tf index 3fa7c1f442..b7b7310ab7 100644 --- a/local.tf +++ b/local.tf @@ -75,7 +75,8 @@ locals { root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device launch_template_id = null # The id of the launch template used for managed node_groups - launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group + launch_template_version = "$Latest" # The latest version of the launch template to use in the autoscaling group + update_default_version = false # Update the autoscaling group launch template's default version upon each update launch_template_placement_tenancy = "default" # The placement tenancy for instances launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. root_encrypted = false # Whether the volume should be encrypted or not diff --git a/workers_launch_template.tf b/workers_launch_template.tf index f22a48f294..0576286b97 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -141,7 +141,13 @@ resource "aws_autoscaling_group" "workers_launch_template" { version = lookup( var.worker_groups_launch_template[count.index], "launch_template_version", - local.workers_group_defaults["launch_template_version"], + lookup( + var.worker_groups_launch_template[count.index], + "launch_template_version", + local.workers_group_defaults["launch_template_version"] + ) == "$Latest" + ? aws_launch_template.workers_launch_template.*.latest_version[count.index] + : aws_launch_template.workers_launch_template.*.default_version[count.index] ) } @@ -169,7 +175,13 @@ resource "aws_autoscaling_group" "workers_launch_template" { version = lookup( var.worker_groups_launch_template[count.index], "launch_template_version", - local.workers_group_defaults["launch_template_version"], + lookup( + var.worker_groups_launch_template[count.index], + "launch_template_version", + local.workers_group_defaults["launch_template_version"] + ) == "$Latest" + ? aws_launch_template.workers_launch_template.*.latest_version[count.index] + : aws_launch_template.workers_launch_template.*.default_version[count.index] ) } } @@ -278,6 +290,12 @@ resource "aws_launch_template" "workers_launch_template" { count.index, )}" + update_default_version = lookup( + var.worker_groups_launch_template[count.index], + "update_default_version", + local.workers_group_defaults["update_default_version"], + ) + network_interfaces { associate_public_ip_address = lookup( var.worker_groups_launch_template[count.index], From 06e9078bf4baedba362f07dcfa0f3fb65e943899 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 19 May 2021 16:47:40 +0200 Subject: [PATCH 305/962] feat: Search for Windows or Linux AMIs only if they are needed (#1371) --- README.md | 1 + data.tf | 4 +++ examples/basic/main.tf | 2 +- examples/fargate/main.tf | 2 +- examples/instance_refresh/main.tf | 2 +- examples/irsa/main.tf | 2 +- examples/launch_templates/main.tf | 2 +- .../main.tf | 2 +- examples/managed_node_groups/main.tf | 2 +- examples/secrets_encryption/main.tf | 2 +- examples/spot_instances/main.tf | 2 +- local.tf | 27 ++++++++++--------- outputs.tf | 7 ++++- 13 files changed, 35 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9b31790031..141a53e4bb 100644 --- a/README.md +++ b/README.md @@ -316,6 +316,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | [workers\_asg\_arns](#output\_workers\_asg\_arns) | IDs of the autoscaling groups containing workers. | | [workers\_asg\_names](#output\_workers\_asg\_names) | Names of the autoscaling groups containing workers. | | [workers\_default\_ami\_id](#output\_workers\_default\_ami\_id) | ID of the default worker group AMI | +| [workers\_default\_ami\_id\_windows](#output\_workers\_default\_ami\_id\_windows) | ID of the default Windows worker group AMI | | [workers\_launch\_template\_arns](#output\_workers\_launch\_template\_arns) | ARNs of the worker launch templates. | | [workers\_launch\_template\_ids](#output\_workers\_launch\_template\_ids) | IDs of the worker launch templates. | | [workers\_launch\_template\_latest\_versions](#output\_workers\_launch\_template\_latest\_versions) | Latest versions of the worker launch templates. | diff --git a/data.tf b/data.tf index a574e83bc6..cc70fd509b 100644 --- a/data.tf +++ b/data.tf @@ -14,6 +14,8 @@ data "aws_iam_policy_document" "workers_assume_role_policy" { } data "aws_ami" "eks_worker" { + count = local.worker_has_linux_ami ? 1 : 0 + filter { name = "name" values = [local.worker_ami_name_filter] @@ -25,6 +27,8 @@ data "aws_ami" "eks_worker" { } data "aws_ami" "eks_worker_windows" { + count = local.worker_has_windows_ami ? 1 : 0 + filter { name = "name" values = [local.worker_ami_name_filter_windows] diff --git a/examples/basic/main.tf b/examples/basic/main.tf index a1e443290a..c6257a25eb 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -103,7 +103,7 @@ module "vpc" { module "eks" { source = "../.." cluster_name = local.cluster_name - cluster_version = "1.17" + cluster_version = "1.20" subnets = module.vpc.private_subnets tags = { diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf index 167805d185..72514404ce 100644 --- a/examples/fargate/main.tf +++ b/examples/fargate/main.tf @@ -60,7 +60,7 @@ module "vpc" { module "eks" { source = "../.." cluster_name = local.cluster_name - cluster_version = "1.17" + cluster_version = "1.20" subnets = module.vpc.private_subnets tags = { diff --git a/examples/instance_refresh/main.tf b/examples/instance_refresh/main.tf index dd7af88a2e..1883ecc70c 100644 --- a/examples/instance_refresh/main.tf +++ b/examples/instance_refresh/main.tf @@ -228,7 +228,7 @@ resource "aws_autoscaling_lifecycle_hook" "aws_node_termination_handler" { module "eks" { source = "../.." cluster_name = local.cluster_name - cluster_version = "1.19" + cluster_version = "1.20" subnets = module.vpc.public_subnets vpc_id = module.vpc.vpc_id enable_irsa = true diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index f881547262..820eff0c01 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -39,7 +39,7 @@ module "vpc" { module "eks" { source = "../.." cluster_name = local.cluster_name - cluster_version = "1.17" + cluster_version = "1.20" subnets = module.vpc.public_subnets vpc_id = module.vpc.vpc_id enable_irsa = true diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 98cac244bf..fe5df294ce 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -43,7 +43,7 @@ module "vpc" { module "eks" { source = "../.." cluster_name = local.cluster_name - cluster_version = "1.17" + cluster_version = "1.20" subnets = module.vpc.public_subnets vpc_id = module.vpc.vpc_id diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf index 95c43a38f0..8d99dcfee5 100644 --- a/examples/launch_templates_with_managed_node_groups/main.tf +++ b/examples/launch_templates_with_managed_node_groups/main.tf @@ -50,7 +50,7 @@ module "vpc" { module "eks" { source = "../.." cluster_name = local.cluster_name - cluster_version = "1.17" + cluster_version = "1.20" subnets = module.vpc.private_subnets vpc_id = module.vpc.vpc_id diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index c2e910e4a5..32db73cf09 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -56,7 +56,7 @@ module "vpc" { module "eks" { source = "../.." cluster_name = local.cluster_name - cluster_version = "1.17" + cluster_version = "1.20" subnets = module.vpc.private_subnets tags = { diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index 4612700f20..9aebd4cbe5 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -60,7 +60,7 @@ module "vpc" { module "eks" { source = "../.." cluster_name = local.cluster_name - cluster_version = "1.17" + cluster_version = "1.20" subnets = module.vpc.private_subnets cluster_encryption_config = [ diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 67215d16a9..fb2ad23ee9 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -43,7 +43,7 @@ module "vpc" { module "eks" { source = "../.." cluster_name = local.cluster_name - cluster_version = "1.17" + cluster_version = "1.20" subnets = module.vpc.public_subnets vpc_id = module.vpc.vpc_id diff --git a/local.tf b/local.tf index b7b7310ab7..cbf040c936 100644 --- a/local.tf +++ b/local.tf @@ -7,14 +7,17 @@ locals { worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] - default_ami_id_linux = coalesce(local.workers_group_defaults.ami_id, data.aws_ami.eks_worker.id) - default_ami_id_windows = coalesce(local.workers_group_defaults.ami_id_windows, data.aws_ami.eks_worker_windows.id) + default_ami_id_linux = local.workers_group_defaults.ami_id != "" ? local.workers_group_defaults.ami_id : concat(data.aws_ami.eks_worker.*.id, [""])[0] + default_ami_id_windows = local.workers_group_defaults.ami_id_windows != "" ? local.workers_group_defaults.ami_id_windows : concat(data.aws_ami.eks_worker_windows.*.id, [""])[0] kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name worker_group_count = length(var.worker_groups) worker_group_launch_template_count = length(var.worker_groups_launch_template) + worker_has_linux_ami = length([for x in concat(var.worker_groups, var.worker_groups_launch_template) : x if lookup(x, "platform", "linux") == "linux"]) > 0 + worker_has_windows_ami = length([for x in concat(var.worker_groups, var.worker_groups_launch_template) : x if lookup(x, "platform", "linux") == "windows"]) > 0 + worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" # Windows nodes are available from k8s 1.14. If cluster version is less than 1.14, fix ami filter to some constant to not fail on 'terraform plan'. worker_ami_name_filter_windows = (var.worker_ami_name_filter_windows != "" ? @@ -72,16 +75,16 @@ locals { additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. # Settings for launch templates - root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. - root_kms_key_id = "" # The KMS key to use when encrypting the root storage device - launch_template_id = null # The id of the launch template used for managed node_groups - launch_template_version = "$Latest" # The latest version of the launch template to use in the autoscaling group - update_default_version = false # Update the autoscaling group launch template's default version upon each update - launch_template_placement_tenancy = "default" # The placement tenancy for instances - launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. - root_encrypted = false # Whether the volume should be encrypted or not - eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) - cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs + root_block_device_name = concat(data.aws_ami.eks_worker.*.root_device_name, [""])[0] # Root device name for workers. If non is provided, will assume default AMI was used. + root_kms_key_id = "" # The KMS key to use when encrypting the root storage device + launch_template_id = null # The id of the launch template used for managed node_groups + launch_template_version = "$Latest" # The latest version of the launch template to use in the autoscaling group + update_default_version = false # Update the autoscaling group launch template's default version upon each update + launch_template_placement_tenancy = "default" # The placement tenancy for instances + launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. + root_encrypted = false # Whether the volume should be encrypted or not + eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) + cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs market_type = null metadata_http_endpoint = "enabled" # The state of the metadata service: enabled, disabled. metadata_http_tokens = "optional" # If session tokens are required: optional, required. diff --git a/outputs.tf b/outputs.tf index b751fb59b1..f6c53513d9 100644 --- a/outputs.tf +++ b/outputs.tf @@ -116,7 +116,12 @@ output "workers_user_data" { output "workers_default_ami_id" { description = "ID of the default worker group AMI" - value = data.aws_ami.eks_worker.id + value = local.default_ami_id_linux +} + +output "workers_default_ami_id_windows" { + description = "ID of the default Windows worker group AMI" + value = local.default_ami_id_windows } output "workers_launch_template_ids" { From 70f615095623ad19e527d633c58d08df7a27d869 Mon Sep 17 00:00:00 2001 From: Andrey Devyatkin Date: Wed, 19 May 2021 21:17:18 +0200 Subject: [PATCH 306/962] docs: Add Bottlerocket example (#1296) Signed-off-by: Andrey Devyatkin --- examples/bottlerocket/README.md | 7 ++++ examples/bottlerocket/data.tf | 22 ++++++++++ examples/bottlerocket/main.tf | 64 +++++++++++++++++++++++++++++ examples/bottlerocket/userdata.toml | 24 +++++++++++ examples/bottlerocket/variables.tf | 17 ++++++++ 5 files changed, 134 insertions(+) create mode 100644 examples/bottlerocket/README.md create mode 100644 examples/bottlerocket/data.tf create mode 100644 examples/bottlerocket/main.tf create mode 100644 examples/bottlerocket/userdata.toml create mode 100644 examples/bottlerocket/variables.tf diff --git a/examples/bottlerocket/README.md b/examples/bottlerocket/README.md new file mode 100644 index 0000000000..cc73eb7634 --- /dev/null +++ b/examples/bottlerocket/README.md @@ -0,0 +1,7 @@ +# AWS Bottlerocket based nodes + +This is a minimalistic example that shows how to use functionality of this module to deploy +nodes based on [AWS Bottlerocket container OS](https://github.com/bottlerocket-os/bottlerocket) + +Example is minimalistic by purpose - it shows what knobs to turn to make Bottlerocket work. +Do not use default VPC for your workloads deployment. \ No newline at end of file diff --git a/examples/bottlerocket/data.tf b/examples/bottlerocket/data.tf new file mode 100644 index 0000000000..bf380b382c --- /dev/null +++ b/examples/bottlerocket/data.tf @@ -0,0 +1,22 @@ +data "aws_ami" "bottlerocket_ami" { + most_recent = true + owners = ["amazon"] + filter { + name = "name" + values = ["bottlerocket-aws-k8s-${var.k8s_version}-x86_64-*"] + } +} + +data "aws_region" "current" {} + +data "aws_vpc" "default" { + default = true +} + +data "aws_subnet_ids" "default" { + vpc_id = data.aws_vpc.default.id +} + +data "aws_iam_policy" "ssm" { + arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} \ No newline at end of file diff --git a/examples/bottlerocket/main.tf b/examples/bottlerocket/main.tf new file mode 100644 index 0000000000..86166df0e5 --- /dev/null +++ b/examples/bottlerocket/main.tf @@ -0,0 +1,64 @@ +terraform { + required_version = ">= 0.13.0" +} + +resource "tls_private_key" "nodes" { + algorithm = "RSA" +} + +resource "aws_key_pair" "nodes" { + key_name = "bottlerocket-nodes" + public_key = tls_private_key.nodes.public_key_openssh +} + +module "eks" { + source = "../.." + cluster_name = "bottlerocket" + cluster_version = var.k8s_version + subnets = data.aws_subnet_ids.default.ids + + vpc_id = data.aws_vpc.default.id + + write_kubeconfig = false + manage_aws_auth = false + + worker_groups_launch_template = [ + { + name = "bottlerocket-nodes" + # passing bottlerocket ami id + ami_id = data.aws_ami.bottlerocket_ami.id + instance_type = "t3a.small" + asg_desired_capacity = 2 + key_name = aws_key_pair.nodes.key_name + + # Since we are using default VPC there is no NAT gateway so we need to + # attach public ip to nodes so they can reach k8s API server + # do not repeat this at home (i.e. production) + public_ip = true + + # This section overrides default userdata template to pass bottlerocket + # specific user data + userdata_template_file = "${path.module}/userdata.toml" + # we are using this section to pass additional arguments for + # userdata template rendering + userdata_template_extra_args = { + enable_admin_container = var.enable_admin_container + enable_control_container = var.enable_control_container + aws_region = data.aws_region.current.name + } + # example of k8s/kubelet configuration via additional_userdata + additional_userdata = < Date: Wed, 19 May 2021 12:23:35 -0700 Subject: [PATCH 307/962] docs: Fix cluster autoscaler tags in IRSA example (#1204) --- examples/irsa/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index 820eff0c01..e6c9fa4018 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -58,7 +58,7 @@ module "eks" { { "key" = "k8s.io/cluster-autoscaler/${local.cluster_name}" "propagate_at_launch" = "false" - "value" = "true" + "value" = "owned" } ] } From ff4450e297126f321f7dbd5d0f3660020dc8879f Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 19 May 2021 22:51:03 +0200 Subject: [PATCH 308/962] chore: Update changelog (#1377) --- CHANGELOG.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9b26f1394..135fb0c412 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,23 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v16.1.0] - 2021-05-19 +FEATURES: +- Search for Windows or Linux AMIs only if they are needed ([#1371](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1371)) + +BUG FIXES: +- Set an ASG's launch template version to an explicit version to automatically trigger instance refresh ([#1370](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1370)) +- Add description for private API ingress Security Group Rule ([#1299](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1299)) + +DOCS: +- Fix cluster autoscaler tags in IRSA example ([#1204](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1204)) +- Add Bottlerocket example ([#1296](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1296)) + +NOTES: +- Set an ASG's launch template version to an explicit version automatically. This will ensure that an instance refresh will be triggered whenever the launch template changes. The default `launch_template_version` is now used to determine the latest or default version of the created launch template for self-managed worker groups. + + ## [v16.0.1] - 2021-05-19 BUG FIXES: @@ -310,7 +327,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.1...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.1.0...HEAD +[v16.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.1...v16.1.0 [v16.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.0...v16.0.1 [v16.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.2.0...v16.0.0 [v15.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.1.0...v15.2.0 From f024da827687d840a2ad4301d6d997be33dcd785 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 20 May 2021 13:08:36 +0200 Subject: [PATCH 309/962] docs: Update license to Apache 2 License (#1375) --- LICENSE | 183 +++++++++++++++++++++++++++++++++++++++++++++++++++--- README.md | 2 +- 2 files changed, 175 insertions(+), 10 deletions(-) diff --git a/LICENSE b/LICENSE index 51fca54c2a..d9a10c0d8e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,11 +1,176 @@ -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md index 141a53e4bb..db57ac8cfe 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ Many thanks to [the contributors listed here](https://github.com/terraform-aws-m ## License -MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. +Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. ## Requirements From 208f3207337bc55e01b5477d97ff3c05a65737d2 Mon Sep 17 00:00:00 2001 From: Vladimir Kurlenya Date: Fri, 21 May 2021 11:04:38 +0300 Subject: [PATCH 310/962] feat: Add ability to forcefully update nodes in managed node groups (#1380) --- modules/node_groups/README.md | 1 + modules/node_groups/node_groups.tf | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index b7443cb326..a4d45024b5 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -21,6 +21,7 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | ami\_release\_version | AMI version of workers | string | Provider default behavior | | ami\_type | AMI Type. See Terraform or AWS docs | string | Provider default behavior | | capacity\_type | Type of instance capacity to provision. Options are `ON_DEMAND` and `SPOT` | string | Provider default behavior | +| force\_update\_version | Force version update if existing pods are unable to be drained due to a pod disruption budget issue. | bool | Provider default behavior | | desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` | | disk\_size | Workers' disk size | number | Provider default behavior | | disk\_type | Workers' disk type. Require `create_launch_template` to be `true`| number | `gp3` | diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 8b9904cb17..afb65059f8 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -13,11 +13,12 @@ resource "aws_eks_node_group" "workers" { min_size = each.value["min_capacity"] } - ami_type = lookup(each.value, "ami_type", null) - disk_size = each.value["launch_template_id"] != null || each.value["create_launch_template"] ? null : lookup(each.value, "disk_size", null) - instance_types = !each.value["set_instance_types_on_lt"] ? each.value["instance_types"] : null - release_version = lookup(each.value, "ami_release_version", null) - capacity_type = lookup(each.value, "capacity_type", null) + ami_type = lookup(each.value, "ami_type", null) + disk_size = each.value["launch_template_id"] != null || each.value["create_launch_template"] ? null : lookup(each.value, "disk_size", null) + instance_types = !each.value["set_instance_types_on_lt"] ? each.value["instance_types"] : null + release_version = lookup(each.value, "ami_release_version", null) + capacity_type = lookup(each.value, "capacity_type", null) + force_update_version = lookup(each.value, "force_update_version", null) dynamic "remote_access" { for_each = each.value["key_name"] != "" && each.value["launch_template_id"] == null && !each.value["create_launch_template"] ? [{ From 84e7308c6df5a4eb69cb8bcc29f7db972e3a2585 Mon Sep 17 00:00:00 2001 From: SimonDreher <60104225+SimonDreher@users.noreply.github.com> Date: Mon, 24 May 2021 11:25:24 +0200 Subject: [PATCH 311/962] docs: Add `iam:DeletePolicyVersion` IAM permissions (#1229) --- docs/iam-permissions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index c175b16429..6e0fb2a1fd 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -109,6 +109,7 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "iam:DeleteInstanceProfile", "iam:DeleteOpenIDConnectProvider", "iam:DeletePolicy", + "iam:DeletePolicyVersion" "iam:DeleteRole", "iam:DeleteRolePolicy", "iam:DeleteServiceLinkedRole", From 7898da2ff73a9355774b7c871ebed8270fe7c7cc Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 24 May 2021 21:54:28 +0200 Subject: [PATCH 312/962] fix: Bump `terraform-provider-http` required version to 2.4.1 to avoid TLS Cert Pool issue on Windows (#1387) --- README.md | 4 ++-- versions.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index db57ac8cfe..f0cf1cda04 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.37.0 | -| [http](#requirement\_http) | >= 2.4.0 | +| [http](#requirement\_http) | >= 2.4.1 | | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -154,7 +154,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.37.0 | -| [http](#provider\_http) | >= 2.4.0 | +| [http](#provider\_http) | >= 2.4.1 | | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | diff --git a/versions.tf b/versions.tf index 7fc51f80b0..877ef298bd 100644 --- a/versions.tf +++ b/versions.tf @@ -8,7 +8,7 @@ terraform { kubernetes = ">= 1.11.1" http = { source = "terraform-aws-modules/http" - version = ">= 2.4.0" + version = ">= 2.4.1" } } } From c039e1e7a2e1fb7b27aa31f6f74dd4f03f4f2a8f Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 24 May 2021 23:13:23 +0200 Subject: [PATCH 313/962] chore: Update changelog (#1388) --- CHANGELOG.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 135fb0c412..5b3071d153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,18 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v16.2.0] - 2021-05-24 +FEATURES: +- Add ability to forcefully update nodes in managed node groups ([#1380](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1380)) + +BUG FIXES: +- Bump `terraform-provider-http` required version to 2.4.1 to avoid TLS Cert Pool issue on Windows ([#1387](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1387)) + +DOCS: +- Update license to Apache 2 License ([#1375](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1375)) + + ## [v16.1.0] - 2021-05-19 FEATURES: @@ -327,7 +339,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.1.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.2.0...HEAD +[v16.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.1.0...v16.2.0 [v16.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.1...v16.1.0 [v16.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.0...v16.0.1 [v16.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v15.2.0...v16.0.0 From 80f83fe4afd60e1ab2c19a16aa35dc7335c0f0a1 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 24 May 2021 23:40:59 +0200 Subject: [PATCH 314/962] chore: Remove CI tests from checklist in PR template (#1383) --- .github/PULL_REQUEST_TEMPLATE.md | 1 - .github/workflows/changelog-check.yaml | 38 +++++++++++++++++++------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6a10d8fa5f..ce48f0220a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,5 +6,4 @@ Please explain the changes you made here and link to any relevant issues. ### Checklist -- [ ] CI tests are passing - [ ] README.md has been updated after any changes to variables and outputs. See https://github.com/terraform-aws-modules/terraform-aws-eks/#doc-generation diff --git a/.github/workflows/changelog-check.yaml b/.github/workflows/changelog-check.yaml index 34fe6056c3..c884de9ec9 100644 --- a/.github/workflows/changelog-check.yaml +++ b/.github/workflows/changelog-check.yaml @@ -1,20 +1,38 @@ name: CHANGELOG Checks on: + push: + branches: + - master pull_request: paths: - CHANGELOG.md + pull_request_target: jobs: - changelog-check: - name: CHANGELOG Check + comment: + name: Comment runs-on: ubuntu-latest steps: + - name: Find Existing PR Comment + id: prc + uses: peter-evans/find-comment@v1 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: "The `CHANGELOG.md` file contents are handled by the maintainers during merge." + - name: PR Comment + if: ${{ steps.prc.outputs.comment-id == '' }} + uses: peter-evans/create-or-update-comment@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + Thank you for your contribution! + + The `CHANGELOG.md` file contents are handled by the maintainers during merge. This is to prevent pull request merge conflicts. + Please see the Contributing Guide for additional pull request review items. + + Remove any changes to the `CHANGELOG.md` file and commit them in this pull request. - name: Fail the check if changelog change - run: |- - echo "Thank you for your contribution!" - echo "" - echo "The 'CHANGELOG.md' file contents are handled by the maintainers during merge. This is to prevent pull request merge conflicts." - echo "Please see the Contributing Guide for additional pull request review items." - echo "" - echo "Remove any changes to the 'CHANGELOG.md' file and commit them in this pull request." - exit 1 + run: exit 1 From 45f3b2f3f3643e723833d7146702543b28ea2ef3 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 25 May 2021 10:56:57 +0200 Subject: [PATCH 315/962] chore: check for changelog only when CHANGELOG.md change (#1390) --- .github/workflows/changelog-check.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/changelog-check.yaml b/.github/workflows/changelog-check.yaml index c884de9ec9..c22baef4cc 100644 --- a/.github/workflows/changelog-check.yaml +++ b/.github/workflows/changelog-check.yaml @@ -1,12 +1,15 @@ name: CHANGELOG Checks on: - push: - branches: - - master pull_request: paths: - CHANGELOG.md + branches-ignore: + - releases/** pull_request_target: + paths: + - CHANGELOG.md + branches-ignore: + - releases/** jobs: comment: From 1f22d24df63accd4446aa2079eeac9226741bde4 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 25 May 2021 11:06:04 +0200 Subject: [PATCH 316/962] fix: Add back `depends_on` for `data.wait_for_cluster` (#1389) --- cluster.tf | 32 ++++++++++++-------------------- data.tf | 12 ++++++++++++ 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/cluster.tf b/cluster.tf index 7acfd3367b..94581a4de3 100644 --- a/cluster.tf +++ b/cluster.tf @@ -52,26 +52,6 @@ resource "aws_eks_cluster" "this" { ] } -resource "aws_security_group_rule" "cluster_private_access" { - description = "Allow private K8S API ingress from custom source." - count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access ? 1 : 0 - type = "ingress" - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = var.cluster_endpoint_private_access_cidrs - - security_group_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id -} - - -data "http" "wait_for_cluster" { - count = var.create_eks && var.manage_aws_auth ? 1 : 0 - url = format("%s/healthz", aws_eks_cluster.this[0].endpoint) - ca_certificate = base64decode(coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0]) - timeout = 300 -} - resource "aws_security_group" "cluster" { count = var.cluster_create_security_group && var.create_eks ? 1 : 0 name_prefix = var.cluster_name @@ -107,6 +87,18 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { type = "ingress" } +resource "aws_security_group_rule" "cluster_private_access" { + description = "Allow private K8S API ingress from custom source." + count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access ? 1 : 0 + type = "ingress" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = var.cluster_endpoint_private_access_cidrs + + security_group_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id +} + resource "aws_iam_role" "cluster" { count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 name_prefix = var.cluster_iam_role_name != "" ? null : var.cluster_name diff --git a/data.tf b/data.tf index cc70fd509b..a7c00eb563 100644 --- a/data.tf +++ b/data.tf @@ -83,3 +83,15 @@ data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instanc } data "aws_partition" "current" {} + +data "http" "wait_for_cluster" { + count = var.create_eks && var.manage_aws_auth ? 1 : 0 + url = format("%s/healthz", aws_eks_cluster.this[0].endpoint) + ca_certificate = base64decode(coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0]) + timeout = 300 + + depends_on = [ + aws_eks_cluster.this, + aws_security_group_rule.cluster_private_access, + ] +} From 9571a19028b390f5913be8f016ebd9240220aea3 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 25 May 2021 11:11:00 +0200 Subject: [PATCH 317/962] chore: Check for changelog on `pull_request_target` only (#1391) --- .github/workflows/changelog-check.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/changelog-check.yaml b/.github/workflows/changelog-check.yaml index c22baef4cc..0eee4fdbe0 100644 --- a/.github/workflows/changelog-check.yaml +++ b/.github/workflows/changelog-check.yaml @@ -1,10 +1,5 @@ name: CHANGELOG Checks on: - pull_request: - paths: - - CHANGELOG.md - branches-ignore: - - releases/** pull_request_target: paths: - CHANGELOG.md From 6aac9c49616d4b2dc350145a8f1cefa1b7f06515 Mon Sep 17 00:00:00 2001 From: Jason Witkowski Date: Thu, 27 May 2021 15:46:38 -0400 Subject: [PATCH 318/962] feat: Add instance store volume option for instances with local disk (#1213) --- local.tf | 93 ++++++++++++++++++++------------------ workers_launch_template.tf | 12 +++++ 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/local.tf b/local.tf index cbf040c936..11d16b7b87 100644 --- a/local.tf +++ b/local.tf @@ -29,51 +29,54 @@ locals { policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" workers_group_defaults_defaults = { - name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. - tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. - ami_id = "" # AMI ID for the eks linux based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. - ami_id_windows = "" # AMI ID for the eks windows based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. - asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. - asg_max_size = "3" # Maximum worker capacity in the autoscaling group. - asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. - asg_force_delete = false # Enable forced deletion for the autoscaling group. - asg_initial_lifecycle_hooks = [] # Initial lifecycle hook for the autoscaling group. - default_cooldown = null # The amount of time, in seconds, after a scaling activity completes before another scaling activity can start. - health_check_type = null # Controls how health checking is done. Valid values are "EC2" or "ELB". - health_check_grace_period = null # Time in seconds after instance comes into service before checking health. - instance_type = "m4.large" # Size of the workers instances. - spot_price = "" # Cost of spot instance. - placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". - root_volume_size = "100" # root volume size of workers instances. - root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp3' (for Launch Template), 'gp2', or 'io1' - root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". - root_volume_throughput = null # The amount of throughput to provision for a gp3 volume. - key_name = "" # The key pair name that should be used for the instances in the autoscaling group - pre_userdata = "" # userdata to pre-append to the default userdata. - userdata_template_file = "" # alternate template to use for userdata - userdata_template_extra_args = {} # Additional arguments to use when expanding the userdata template file - bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI (Amazon Machine Image). - additional_userdata = "" # userdata to append to the default userdata. - ebs_optimized = true # sets whether to use ebs optimization on supported types. - enable_monitoring = true # Enables/disables detailed monitoring. - enclave_support = false # Enables/disables enclave support - public_ip = false # Associate a public ip address with a worker - kubelet_extra_args = "" # This string is passed directly to kubelet if set. Useful for adding labels or taints. - subnets = var.subnets # A list of subnets to place the worker nodes in. i.e. ["subnet-123", "subnet-456", "subnet-789"] - additional_security_group_ids = [] # A list of additional security group ids to include in worker launch config - protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. - iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. - iam_role_id = "local.default_iam_role_id" # A custom IAM role id. Incompatible with iam_instance_profile_name. Literal local.default_iam_role_id will never be used but if iam_role_id is not set, the local.default_iam_role_id interpolation will be used. - suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] - target_group_arns = null # A list of Application LoadBalancer (ALB) target group ARNs to be associated to the autoscaling group - load_balancers = null # A list of Classic LoadBalancer (CLB)'s name to be associated to the autoscaling group - enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] - placement_group = null # The name of the placement group into which to launch the instances, if any. - service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS - termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. - platform = "linux" # Platform of workers. either "linux" or "windows" - additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults - warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. + name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. + tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. + ami_id = "" # AMI ID for the eks linux based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. + ami_id_windows = "" # AMI ID for the eks windows based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. + asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. + asg_max_size = "3" # Maximum worker capacity in the autoscaling group. + asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. + asg_force_delete = false # Enable forced deletion for the autoscaling group. + asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. + default_cooldown = null # The amount of time, in seconds, after a scaling activity completes before another scaling activity can start. + health_check_type = null # Controls how health checking is done. Valid values are "EC2" or "ELB". + health_check_grace_period = null # Time in seconds after instance comes into service before checking health. + instance_type = "m4.large" # Size of the workers instances. + instance_store_virtual_name = "ephemeral0" # "virtual_name" of the instance store volume. + spot_price = "" # Cost of spot instance. + placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". + root_volume_size = "100" # root volume size of workers instances. + root_volume_type = "gp3" # root volume type of workers instances, can be "standard", "gp3", "gp2", or "io1" + root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". + root_volume_throughput = null # The amount of throughput to provision for a gp3 volume. + key_name = "" # The key pair name that should be used for the instances in the autoscaling group + pre_userdata = "" # userdata to pre-append to the default userdata. + userdata_template_file = "" # alternate template to use for userdata + userdata_template_extra_args = {} # Additional arguments to use when expanding the userdata template file + bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI (Amazon Machine Image). + additional_userdata = "" # userdata to append to the default userdata. + ebs_optimized = true # sets whether to use ebs optimization on supported types. + enable_monitoring = true # Enables/disables detailed monitoring. + enclave_support = false # Enables/disables enclave support + public_ip = false # Associate a public ip address with a worker + kubelet_extra_args = "" # This string is passed directly to kubelet if set. Useful for adding labels or taints. + subnets = var.subnets # A list of subnets to place the worker nodes in. i.e. ["subnet-123", "subnet-456", "subnet-789"] + additional_security_group_ids = [] # A list of additional security group ids to include in worker launch config + protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. + iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. + iam_role_id = "local.default_iam_role_id" # A custom IAM role id. Incompatible with iam_instance_profile_name. Literal local.default_iam_role_id will never be used but if iam_role_id is not set, the local.default_iam_role_id interpolation will be used. + suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] + target_group_arns = null # A list of Application LoadBalancer (ALB) target group ARNs to be associated to the autoscaling group + load_balancers = null # A list of Classic LoadBalancer (CLB)'s name to be associated to the autoscaling group + enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] + placement_group = null # The name of the placement group into which to launch the instances, if any. + service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS + termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. + platform = "linux" # Platform of workers. either "linux" or "windows" + additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults + additional_instance_store_volumes = [] # A list of additional instance store (local disk) volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), virtual_name. + warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. + # Settings for launch templates root_block_device_name = concat(data.aws_ami.eks_worker.*.root_device_name, [""])[0] # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 0576286b97..3710b0a2b3 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -518,6 +518,18 @@ resource "aws_launch_template" "workers_launch_template" { } + dynamic "block_device_mappings" { + for_each = lookup(var.worker_groups_launch_template[count.index], "additional_instance_store_volumes", local.workers_group_defaults["additional_instance_store_volumes"]) + content { + device_name = block_device_mappings.value.block_device_name + virtual_name = lookup( + block_device_mappings.value, + "virtual_name", + local.workers_group_defaults["instance_store_virtual_name"], + ) + } + } + tag_specifications { resource_type = "volume" From c43d545158f0752de447f9240f9701b2cdcfe7ee Mon Sep 17 00:00:00 2001 From: alexmnyc Date: Thu, 27 May 2021 16:01:09 -0400 Subject: [PATCH 319/962] feat: Add tags on network interfaces (#1362) --- modules/node_groups/launchtemplate.tf | 14 ++++++++++++++ workers_launch_template.tf | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/modules/node_groups/launchtemplate.tf b/modules/node_groups/launchtemplate.tf index 1da04e2037..7b50350dd2 100644 --- a/modules/node_groups/launchtemplate.tf +++ b/modules/node_groups/launchtemplate.tf @@ -98,6 +98,20 @@ resource "aws_launch_template" "workers" { ) } + # Supplying custom tags to EKS instances ENI's + tag_specifications { + resource_type = "network-interface" + + tags = merge( + var.tags, + lookup(var.node_groups_defaults, "additional_tags", {}), + lookup(var.node_groups[each.key], "additional_tags", {}), + { + Name = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) + } + ) + } + # Tag the LT itself tags = merge( var.tags, diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 3710b0a2b3..aaea9616cc 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -563,6 +563,24 @@ resource "aws_launch_template" "workers_launch_template" { ) } + tag_specifications { + resource_type = "network-interface" + + tags = merge( + { + "Name" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( + var.worker_groups_launch_template[count.index], + "name", + count.index, + )}-eks_asg" + }, + { for tag_key, tag_value in var.tags : + tag_key => tag_value + if tag_key != "Name" && !contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) + } + ) + } + tags = var.tags lifecycle { From 7062cd6f9438428d9a66bb66455832691079384f Mon Sep 17 00:00:00 2001 From: Alexandre Hamez Date: Thu, 27 May 2021 22:04:48 +0200 Subject: [PATCH 320/962] docs: Add KMS aliases handling to IAM permissions (#1288) --- docs/iam-permissions.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index 6e0fb2a1fd..27f88f22e1 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -134,11 +134,14 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "logs:ListTagsLogGroup", "logs:PutRetentionPolicy", // Following permissions for working with secrets_encryption example + "kms:CreateAlias", "kms:CreateGrant", "kms:CreateKey", + "kms:DeleteAlias", "kms:DescribeKey", "kms:GetKeyPolicy", "kms:GetKeyRotationStatus", + "kms:ListAliases", "kms:ListResourceTags", "kms:ScheduleKeyDeletion" ], From 4a9fc3af1176ea01c703058bc209c7d178279669 Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Thu, 27 May 2021 13:29:17 -0700 Subject: [PATCH 321/962] feat: Kubeconfig file should not be world or group readable by default (#1114) Co-authored-by: Thierno IB. BARRY --- README.md | 1 + kubectl.tf | 2 +- variables.tf | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f0cf1cda04..214fe15d06 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [kubeconfig\_aws\_authenticator\_command](#input\_kubeconfig\_aws\_authenticator\_command) | Command to use to fetch AWS EKS credentials. | `string` | `"aws-iam-authenticator"` | no | | [kubeconfig\_aws\_authenticator\_command\_args](#input\_kubeconfig\_aws\_authenticator\_command\_args) | Default arguments passed to the authenticator command. Defaults to [token -i $cluster\_name]. | `list(string)` | `[]` | no | | [kubeconfig\_aws\_authenticator\_env\_variables](#input\_kubeconfig\_aws\_authenticator\_env\_variables) | Environment variables that should be used when executing the authenticator. e.g. { AWS\_PROFILE = "eks"}. | `map(string)` | `{}` | no | +| [kubeconfig\_file\_permission](#input\_kubeconfig\_file\_permission) | File permission of the Kubectl config file containing cluster configuration saved to `config_output_path.` | `string` | `"0600"` | no | | [kubeconfig\_name](#input\_kubeconfig\_name) | Override the default name used for items kubeconfig. | `string` | `""` | no | | [manage\_aws\_auth](#input\_manage\_aws\_auth) | Whether to apply the aws-auth configmap file. | `bool` | `true` | no | | [manage\_cluster\_iam\_resources](#input\_manage\_cluster\_iam\_resources) | Whether to let the module manage cluster IAM resources. If set to false, cluster\_iam\_role\_name must be specified. | `bool` | `true` | no | diff --git a/kubectl.tf b/kubectl.tf index a83f0f95e7..6e12ea77a3 100644 --- a/kubectl.tf +++ b/kubectl.tf @@ -2,6 +2,6 @@ resource "local_file" "kubeconfig" { count = var.write_kubeconfig && var.create_eks ? 1 : 0 content = local.kubeconfig filename = substr(var.config_output_path, -1, 1) == "/" ? "${var.config_output_path}kubeconfig_${var.cluster_name}" : var.config_output_path - file_permission = "0644" + file_permission = var.kubeconfig_file_permission directory_permission = "0755" } diff --git a/variables.tf b/variables.tf index a48093d577..e9f9d6ccdb 100644 --- a/variables.tf +++ b/variables.tf @@ -38,6 +38,12 @@ variable "config_output_path" { default = "./" } +variable "kubeconfig_file_permission" { + description = "File permission of the Kubectl config file containing cluster configuration saved to `config_output_path.`" + type = string + default = "0600" +} + variable "write_kubeconfig" { description = "Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`." type = bool From 876536209304a4cb7d7f4e189e527dec80e5b3d5 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 27 May 2021 22:59:09 +0200 Subject: [PATCH 322/962] feat: Rename `config_output_path` into `kubeconfig_output_path` for naming consistency (#1399) BREAKING CHANGES: The variable `config_output_path` is renamed into `kubeconfig_output_path` for naming consistency. Please upgrade your configuration accordingly. NOTES: The kubeconfig file permission is not world and group readable anymore. The default permission is now `600`. This value can be changed with the variable `var.kubeconfig_file_permission`. --- README.md | 6 +++--- kubectl.tf | 2 +- variables.tf | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 214fe15d06..9c792d6e24 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,6 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | -| [config\_output\_path](#input\_config\_output\_path) | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | | [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | [create\_fargate\_pod\_execution\_role](#input\_create\_fargate\_pod\_execution\_role) | Controls if the EKS Fargate pod execution IAM role should be created. | `bool` | `true` | no | | [eks\_oidc\_root\_ca\_thumbprint](#input\_eks\_oidc\_root\_ca\_thumbprint) | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | @@ -251,8 +250,9 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [kubeconfig\_aws\_authenticator\_command](#input\_kubeconfig\_aws\_authenticator\_command) | Command to use to fetch AWS EKS credentials. | `string` | `"aws-iam-authenticator"` | no | | [kubeconfig\_aws\_authenticator\_command\_args](#input\_kubeconfig\_aws\_authenticator\_command\_args) | Default arguments passed to the authenticator command. Defaults to [token -i $cluster\_name]. | `list(string)` | `[]` | no | | [kubeconfig\_aws\_authenticator\_env\_variables](#input\_kubeconfig\_aws\_authenticator\_env\_variables) | Environment variables that should be used when executing the authenticator. e.g. { AWS\_PROFILE = "eks"}. | `map(string)` | `{}` | no | -| [kubeconfig\_file\_permission](#input\_kubeconfig\_file\_permission) | File permission of the Kubectl config file containing cluster configuration saved to `config_output_path.` | `string` | `"0600"` | no | +| [kubeconfig\_file\_permission](#input\_kubeconfig\_file\_permission) | File permission of the Kubectl config file containing cluster configuration saved to `kubeconfig_output_path.` | `string` | `"0600"` | no | | [kubeconfig\_name](#input\_kubeconfig\_name) | Override the default name used for items kubeconfig. | `string` | `""` | no | +| [kubeconfig\_output\_path](#input\_kubeconfig\_output\_path) | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | | [manage\_aws\_auth](#input\_manage\_aws\_auth) | Whether to apply the aws-auth configmap file. | `bool` | `true` | no | | [manage\_cluster\_iam\_resources](#input\_manage\_cluster\_iam\_resources) | Whether to let the module manage cluster IAM resources. If set to false, cluster\_iam\_role\_name must be specified. | `bool` | `true` | no | | [manage\_worker\_iam\_resources](#input\_manage\_worker\_iam\_resources) | Whether to let the module manage worker IAM resources. If set to false, iam\_instance\_profile\_name must be specified for workers. | `bool` | `true` | no | @@ -281,7 +281,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [workers\_egress\_cidrs](#input\_workers\_egress\_cidrs) | List of CIDR blocks that are permitted for workers egress traffic. | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [workers\_group\_defaults](#input\_workers\_group\_defaults) | Override default values for target groups. See workers\_group\_defaults\_defaults in local.tf for valid keys. | `any` | `{}` | no | | [workers\_role\_name](#input\_workers\_role\_name) | User defined workers role name. | `string` | `""` | no | -| [write\_kubeconfig](#input\_write\_kubeconfig) | Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`. | `bool` | `true` | no | +| [write\_kubeconfig](#input\_write\_kubeconfig) | Whether to write a Kubectl config file containing the cluster configuration. Saved to `kubeconfig_output_path`. | `bool` | `true` | no | ## Outputs diff --git a/kubectl.tf b/kubectl.tf index 6e12ea77a3..21021f92c6 100644 --- a/kubectl.tf +++ b/kubectl.tf @@ -1,7 +1,7 @@ resource "local_file" "kubeconfig" { count = var.write_kubeconfig && var.create_eks ? 1 : 0 content = local.kubeconfig - filename = substr(var.config_output_path, -1, 1) == "/" ? "${var.config_output_path}kubeconfig_${var.cluster_name}" : var.config_output_path + filename = substr(var.kubeconfig_output_path, -1, 1) == "/" ? "${var.kubeconfig_output_path}kubeconfig_${var.cluster_name}" : var.kubeconfig_output_path file_permission = var.kubeconfig_file_permission directory_permission = "0755" } diff --git a/variables.tf b/variables.tf index e9f9d6ccdb..8629362783 100644 --- a/variables.tf +++ b/variables.tf @@ -32,20 +32,20 @@ variable "cluster_version" { type = string } -variable "config_output_path" { +variable "kubeconfig_output_path" { description = "Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`." type = string default = "./" } variable "kubeconfig_file_permission" { - description = "File permission of the Kubectl config file containing cluster configuration saved to `config_output_path.`" + description = "File permission of the Kubectl config file containing cluster configuration saved to `kubeconfig_output_path.`" type = string default = "0600" } variable "write_kubeconfig" { - description = "Whether to write a Kubectl config file containing the cluster configuration. Saved to `config_output_path`." + description = "Whether to write a Kubectl config file containing the cluster configuration. Saved to `kubeconfig_output_path`." type = bool default = true } From 4818043617c756dade359d268cbc0e050fecac19 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Thu, 27 May 2021 23:00:41 +0200 Subject: [PATCH 323/962] feat: Add multiple selectors on the creation of Fargate profile (#1378) BREAKING CHANGES: To support multiple selectors for Fargate profiles, we introduced the `selectors` argument which is a list of map. This will break previous configuration with a single selector `namespace` and `labels`. You'll need to rewrite your configuration to use the `selectors` argument. See [examples](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/examples/fargate/main.tf) dans [docs](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/modules/fargate/README.md) for details. Co-authored-by: Lena --- examples/fargate/main.tf | 28 +++++++++++++++++++--------- modules/fargate/README.md | 5 ++--- modules/fargate/fargate.tf | 10 +++++++--- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf index 72514404ce..0697a58c6b 100644 --- a/examples/fargate/main.tf +++ b/examples/fargate/main.tf @@ -72,15 +72,25 @@ module "eks" { vpc_id = module.vpc.vpc_id fargate_profiles = { - example = { - namespace = "default" - - # Kubernetes labels for selection - # labels = { - # Environment = "test" - # GithubRepo = "terraform-aws-eks" - # GithubOrg = "terraform-aws-modules" - # } + default = { + name = "default" + selectors = [ + { + namespace = "kube-system" + labels = { + k8s-app = "kube-dns" + } + }, + { + namespace = "default" + # Kubernetes labels for selection + # labels = { + # Environment = "test" + # GithubRepo = "terraform-aws-eks" + # GithubOrg = "terraform-aws-modules" + # } + } + ] # using specific subnets instead of all the ones configured in eks # subnets = ["subnet-0ca3e3d1234a56c78"] diff --git a/modules/fargate/README.md b/modules/fargate/README.md index d31e59735b..459c355fe8 100644 --- a/modules/fargate/README.md +++ b/modules/fargate/README.md @@ -11,10 +11,9 @@ Helper submodule to create and manage resources related to `aws_eks_fargate_prof | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | name | Fargate profile name | `string` | Auto generated in the following format `[cluster_name]-fargate-[fargate_profile_map_key]`| no | -| namespace | Kubernetes namespace for selection | `string` | n/a | yes | -| labels | Key-value map of Kubernetes labels for selection | `map(string)` | `{}` | no | -| tags | Key-value map of resource tags. Will be merged with root module tags. | `map(string)` | `var.tags` | no | +| selectors | A list of Kubernetes selectors. See examples/fargate/main.tf for example format. |
list(map({
namespace = string
labels = map(string)
}))
| `[]` | no | | subnets | List of subnet IDs. Will replace the root module subnets. | `list(string)` | `var.subnets` | no | +| tags | Key-value map of resource tags. Will be merged with root module tags. | `map(string)` | `var.tags` | no | ## Requirements diff --git a/modules/fargate/fargate.tf b/modules/fargate/fargate.tf index e88adb63bb..caa73faf88 100644 --- a/modules/fargate/fargate.tf +++ b/modules/fargate/fargate.tf @@ -20,9 +20,13 @@ resource "aws_eks_fargate_profile" "this" { pod_execution_role_arn = local.pod_execution_role_arn subnet_ids = lookup(each.value, "subnets", var.subnets) tags = each.value.tags - selector { - namespace = each.value.namespace - labels = lookup(each.value, "labels", null) + + dynamic "selector" { + for_each = each.value.selectors + content { + namespace = selector.value["namespace"] + labels = lookup(selector.value, "labels", {}) + } } depends_on = [var.eks_depends_on] From 6d7d6f6f5a26a4d3aee527d9ddd087f0fd8bdbe9 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 01:50:16 +0200 Subject: [PATCH 324/962] feat: Drop random pets from Managed Node Groups (#1372) BREAKING CHANGES: We now decided to remove `random_pet` resources in Managed Node Groups (MNG). Those were used to recreate MNG if something change and also simulate the newly added argument `node_group_name_prefix`. But they were causing a lot of troubles. To upgrade the module without recreating your MNG, you will need to explicitly reuse their previous name and set them in your MNG `name` argument. Please see [upgrade docs](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/upgrades.md#upgrade-module-to-v1700-for-managed-node-groups) for more details. --- README.md | 5 +- aws_auth.tf | 2 - data.tf | 6 +- docs/upgrades.md | 60 +++++++++++++++++++ examples/managed_node_groups/main.tf | 2 +- modules/fargate/README.md | 4 +- modules/fargate/versions.tf | 2 +- modules/node_groups/README.md | 24 ++++---- .../{launchtemplate.tf => launch_template.tf} | 19 +++--- modules/node_groups/locals.tf | 10 ++++ modules/node_groups/node_groups.tf | 3 +- modules/node_groups/random.tf | 27 --------- modules/node_groups/versions.tf | 3 +- versions.tf | 3 +- 14 files changed, 105 insertions(+), 65 deletions(-) create mode 100644 docs/upgrades.md rename modules/node_groups/{launchtemplate.tf => launch_template.tf} (81%) delete mode 100644 modules/node_groups/random.tf diff --git a/README.md b/README.md index 9c792d6e24..54e92dec46 100644 --- a/README.md +++ b/README.md @@ -143,17 +143,16 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.37.0 | +| [aws](#requirement\_aws) | >= 3.40.0 | | [http](#requirement\_http) | >= 2.4.1 | | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | -| [random](#requirement\_random) | >= 2.1 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.37.0 | +| [aws](#provider\_aws) | >= 3.40.0 | | [http](#provider\_http) | >= 2.4.1 | | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | diff --git a/aws_auth.tf b/aws_auth.tf index f549dcce77..6eb563203d 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -1,5 +1,3 @@ -data "aws_caller_identity" "current" {} - locals { auth_launch_template_worker_roles = [ for index in range(0, var.create_eks ? local.worker_group_launch_template_count : 0) : { diff --git a/data.tf b/data.tf index a7c00eb563..875ca5e8ee 100644 --- a/data.tf +++ b/data.tf @@ -1,3 +1,7 @@ +data "aws_partition" "current" {} + +data "aws_caller_identity" "current" {} + data "aws_iam_policy_document" "workers_assume_role_policy" { statement { sid = "EKSWorkerAssumeRole" @@ -82,8 +86,6 @@ data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instanc ) } -data "aws_partition" "current" {} - data "http" "wait_for_cluster" { count = var.create_eks && var.manage_aws_auth ? 1 : 0 url = format("%s/healthz", aws_eks_cluster.this[0].endpoint) diff --git a/docs/upgrades.md b/docs/upgrades.md new file mode 100644 index 0000000000..66da5b7b75 --- /dev/null +++ b/docs/upgrades.md @@ -0,0 +1,60 @@ +# How to handle the terraform-aws-eks module upgrade + +## Upgrade module to v17.0.0 for Managed Node Groups + +In this release, we now decided to remove random_pet resources in Managed Node Groups (MNG). Those were used to recreate MNG if something changed. But they were causing a lot of issues. To upgrade the module without recreating your MNG, you will need to explicitly reuse their previous name and set them in your MNG `name` argument. + +1. Run `terraform apply` with the module version v16.2.0 +2. Get your worker group names +```shell +~ terraform state show 'module.eks.module.node_groups.aws_eks_node_group.workers["example"]' | grep node_group_name +node_group_name = "test-eks-mwIwsvui-example-sincere-squid" +``` +3. Upgrade your module and configure your node groups to use existing names +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "17.0.0" + + cluster_name = "test-eks-mwIwsvui" + cluster_version = "1.20" + # ... + + node_groups = { + example = { + name = "test-eks-mwIwsvui-example-sincere-squid" + + # ... + } + } + # ... +} +``` +4. Run `terraform plan`, you shoud see that only `random_pets` will be destroyed + +```shell +Terraform will perform the following actions: + + # module.eks.module.node_groups.random_pet.node_groups["example"] will be destroyed + - resource "random_pet" "node_groups" { + - id = "sincere-squid" -> null + - keepers = { + - "ami_type" = "AL2_x86_64" + - "capacity_type" = "SPOT" + - "disk_size" = "50" + - "iam_role_arn" = "arn:aws:iam::123456789123:role/test-eks-mwIwsvui20210527220853611600000009" + - "instance_types" = "t3.large" + - "key_name" = "" + - "node_group_name" = "test-eks-mwIwsvui-example" + - "source_security_group_ids" = "" + - "subnet_ids" = "subnet-xxxxxxxxxxxx|subnet-xxxxxxxxxxxx|subnet-xxxxxxxxxxxx" + } -> null + - length = 2 -> null + - separator = "-" -> null + } + +Plan: 0 to add, 0 to change, 1 to destroy. +``` +5. If everything sounds good to you, run `terraform apply` + +After the first apply, we recommand you to create a new node group and let the module use the `node_group_name_prefix` (by removing the `name` argument) to generate names and collision during node groups re-creation if needed, because the lifce cycle is `create_before_destroy = true`. diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index 32db73cf09..dc799e4203 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -78,7 +78,7 @@ module "eks" { max_capacity = 10 min_capacity = 1 - instance_types = ["m5.large"] + instance_types = ["t3.large"] capacity_type = "SPOT" k8s_labels = { Environment = "test" diff --git a/modules/fargate/README.md b/modules/fargate/README.md index 459c355fe8..858ec968b7 100644 --- a/modules/fargate/README.md +++ b/modules/fargate/README.md @@ -21,13 +21,13 @@ Helper submodule to create and manage resources related to `aws_eks_fargate_prof | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | +| [aws](#requirement\_aws) | >= 3.40.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | +| [aws](#provider\_aws) | >= 3.40.0 | ## Modules diff --git a/modules/fargate/versions.tf b/modules/fargate/versions.tf index 1263d791e8..2051547e51 100644 --- a/modules/fargate/versions.tf +++ b/modules/fargate/versions.tf @@ -2,6 +2,6 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" + aws = ">= 3.40.0" } } diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index a4d45024b5..d9bb496a83 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -21,28 +21,29 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | ami\_release\_version | AMI version of workers | string | Provider default behavior | | ami\_type | AMI Type. See Terraform or AWS docs | string | Provider default behavior | | capacity\_type | Type of instance capacity to provision. Options are `ON_DEMAND` and `SPOT` | string | Provider default behavior | -| force\_update\_version | Force version update if existing pods are unable to be drained due to a pod disruption budget issue. | bool | Provider default behavior | +| create_launch_template | Create and use a default launch template | bool | `false` | | desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` | | disk\_size | Workers' disk size | number | Provider default behavior | | disk\_type | Workers' disk type. Require `create_launch_template` to be `true`| number | `gp3` | +| enable_monitoring | Enables/disables detailed monitoring. Require `create_launch_template` to be `true`| bool | `true` | +| eni_delete | Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) | bool | `true` | +| force\_update\_version | Force version update if existing pods are unable to be drained due to a pod disruption budget issue. | bool | Provider default behavior | | iam\_role\_arn | IAM role ARN for workers | string | `var.default_iam_role_arn` | | instance\_types | Node group's instance type(s). Multiple types can be specified when `capacity_type="SPOT"`. | list | `[var.workers_group_defaults[instance_type]]` | | k8s\_labels | Kubernetes labels | map(string) | No labels applied | | key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` | +| kubelet_extra_args | This string is passed directly to kubelet if set. Useful for adding labels or taints. Require `create_launch_template` to be `true`| string | "" | | launch_template_id | The id of a aws_launch_template to use | string | No LT used | | launch\_template_version | The version of the LT to use | string | none | | max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` | | min\_capacity | Min number of workers | number | `var.workers_group_defaults[asg_min_size]` | -| name | Name of the node group | string | Auto generated | +| name | Name of the node group. If you don't really need this, we recommend you to use `name_prefix` instead. | string | Will use the autogenerate name prefix | +| name_prefix | Name prefix of the node group | string | Auto generated | +| pre_userdata | userdata to pre-append to the default userdata. Require `create_launch_template` to be `true`| string | "" | +| public_ip | Associate a public ip address with a worker. Require `create_launch_template` to be `true`| string | `false` | source\_security\_group\_ids | Source security groups for remote access to workers | list(string) | If key\_name is specified: THE REMOTE ACCESS WILL BE OPENED TO THE WORLD | | subnets | Subnets to contain workers | list(string) | `var.workers_group_defaults[subnets]` | | version | Kubernetes version | string | Provider default behavior | -| create_launch_template | Create and use a default launch template | bool | `false` | -| kubelet_extra_args | This string is passed directly to kubelet if set. Useful for adding labels or taints. Require `create_launch_template` to be `true`| string | "" | -| enable_monitoring | Enables/disables detailed monitoring. Require `create_launch_template` to be `true`| bool | `true` | -| eni_delete | Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) | bool | `true` | -| public_ip | Associate a public ip address with a worker. Require `create_launch_template` to be `true`| string | `false` -| pre_userdata | userdata to pre-append to the default userdata. Require `create_launch_template` to be `true`| string | "" | ## Requirements @@ -50,16 +51,14 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | -| [random](#requirement\_random) | >= 2.1 | +| [aws](#requirement\_aws) | >= 3.40.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | +| [aws](#provider\_aws) | >= 3.40.0 | | [cloudinit](#provider\_cloudinit) | n/a | -| [random](#provider\_random) | >= 2.1 | ## Modules @@ -71,7 +70,6 @@ No modules. |------|------| | [aws_eks_node_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_launch_template.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | -| [random_pet.node_groups](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | | [cloudinit_config.workers_userdata](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | ## Inputs diff --git a/modules/node_groups/launchtemplate.tf b/modules/node_groups/launch_template.tf similarity index 81% rename from modules/node_groups/launchtemplate.tf rename to modules/node_groups/launch_template.tf index 7b50350dd2..6ad2b68f85 100644 --- a/modules/node_groups/launchtemplate.tf +++ b/modules/node_groups/launch_template.tf @@ -1,5 +1,6 @@ data "cloudinit_config" "workers_userdata" { - for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] } + for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] } + gzip = false base64_encode = true boundary = "//" @@ -12,7 +13,6 @@ data "cloudinit_config" "workers_userdata" { kubelet_extra_args = each.value["kubelet_extra_args"] } ) - } } @@ -23,9 +23,10 @@ data "cloudinit_config" "workers_userdata" { # Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, # then the default user-data for bootstrapping a cluster is merged in the copy. resource "aws_launch_template" "workers" { - for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] } - name_prefix = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) - description = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) + for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] } + + name_prefix = local.node_groups_names[each.key] + description = format("EKS Managed Node Group custom LT for %s", local.node_groups_names[each.key]) update_default_version = true block_device_mappings { @@ -79,7 +80,7 @@ resource "aws_launch_template" "workers" { lookup(var.node_groups_defaults, "additional_tags", {}), lookup(var.node_groups[each.key], "additional_tags", {}), { - Name = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) + Name = local.node_groups_names[each.key] } ) } @@ -93,12 +94,12 @@ resource "aws_launch_template" "workers" { lookup(var.node_groups_defaults, "additional_tags", {}), lookup(var.node_groups[each.key], "additional_tags", {}), { - Name = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) + Name = local.node_groups_names[each.key] } ) } - # Supplying custom tags to EKS instances ENI's + # Supplying custom tags to EKS instances ENI's tag_specifications { resource_type = "network-interface" @@ -107,7 +108,7 @@ resource "aws_launch_template" "workers" { lookup(var.node_groups_defaults, "additional_tags", {}), lookup(var.node_groups[each.key], "additional_tags", {}), { - Name = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) + Name = local.node_groups_names[each.key] } ) } diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 3c510d70ab..192b7a01d1 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -25,4 +25,14 @@ locals { var.node_groups_defaults, v, ) if var.create_eks } + + node_groups_names = { for k, v in local.node_groups_expanded : k => lookup( + v, + "name", + lookup( + v, + "name_prefix", + join("-", [var.cluster_name, k]) + ) + ) } } diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index afb65059f8..a0ae456fa7 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -1,7 +1,8 @@ resource "aws_eks_node_group" "workers" { for_each = local.node_groups_expanded - node_group_name = lookup(each.value, "name", join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id])) + node_group_name_prefix = lookup(each.value, "name", null) == null ? local.node_groups_names[each.key] : null + node_group_name = lookup(each.value, "name", null) cluster_name = var.cluster_name node_role_arn = each.value["iam_role_arn"] diff --git a/modules/node_groups/random.tf b/modules/node_groups/random.tf deleted file mode 100644 index c8b36697f4..0000000000 --- a/modules/node_groups/random.tf +++ /dev/null @@ -1,27 +0,0 @@ -resource "random_pet" "node_groups" { - for_each = local.node_groups_expanded - - separator = "-" - length = 2 - - keepers = { - ami_type = lookup(each.value, "ami_type", null) - disk_size = lookup(each.value, "disk_size", null) - capacity_type = lookup(each.value, "capacity_type", null) - iam_role_arn = each.value["iam_role_arn"] - instance_types = join("|", compact( - lookup(each.value, "instance_types", []) - )) - - key_name = each.value["key_name"] - - source_security_group_ids = join("|", compact( - lookup(each.value, "source_security_group_ids", []) - )) - subnet_ids = join("|", each.value["subnets"]) - node_group_name = join("-", [var.cluster_name, each.key]) - launch_template = lookup(each.value, "launch_template_id", null) - } - - depends_on = [var.ng_depends_on] -} diff --git a/modules/node_groups/versions.tf b/modules/node_groups/versions.tf index 9fb30e02ba..2051547e51 100644 --- a/modules/node_groups/versions.tf +++ b/modules/node_groups/versions.tf @@ -2,7 +2,6 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" - random = ">= 2.1" + aws = ">= 3.40.0" } } diff --git a/versions.tf b/versions.tf index 877ef298bd..db42ebeb2f 100644 --- a/versions.tf +++ b/versions.tf @@ -2,9 +2,8 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.37.0" + aws = ">= 3.40.0" local = ">= 1.4" - random = ">= 2.1" kubernetes = ">= 1.11.1" http = { source = "terraform-aws-modules/http" From d7630ef632dcb567403994104c36d05ede66f03b Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 02:34:52 +0200 Subject: [PATCH 325/962] docs: Clarify about the `cluster_endpoint_private_access_cidrs` usage (#1400) --- README.md | 4 ++-- variables.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 54e92dec46..7af8e066c2 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf |------|-------------|------|---------|:--------:| | [attach\_worker\_cni\_policy](#input\_attach\_worker\_cni\_policy) | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | | [aws\_auth\_additional\_labels](#input\_aws\_auth\_additional\_labels) | Additional kubernetes labels applied on aws-auth ConfigMap | `map(string)` | `{}` | no | -| [cluster\_create\_endpoint\_private\_access\_sg\_rule](#input\_cluster\_create\_endpoint\_private\_access\_sg\_rule) | Whether to create security group rules for the access to the Amazon EKS private API server endpoint. | `bool` | `false` | no | +| [cluster\_create\_endpoint\_private\_access\_sg\_rule](#input\_cluster\_create\_endpoint\_private\_access\_sg\_rule) | Whether to create security group rules for the access to the Amazon EKS private API server endpoint. When is `true`, `cluster_endpoint_private_access_cidrs` must be setted. | `bool` | `false` | no | | [cluster\_create\_security\_group](#input\_cluster\_create\_security\_group) | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | | [cluster\_create\_timeout](#input\_cluster\_create\_timeout) | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | | [cluster\_delete\_timeout](#input\_cluster\_delete\_timeout) | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | @@ -228,7 +228,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | | [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster. See examples/secrets\_encryption/main.tf for example format |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | | [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | -| [cluster\_endpoint\_private\_access\_cidrs](#input\_cluster\_endpoint\_private\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS private API server endpoint. | `list(string)` | `null` | no | +| [cluster\_endpoint\_private\_access\_cidrs](#input\_cluster\_endpoint\_private\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS private API server endpoint. To use this `cluster_create_endpoint_private_access_sg_rule` must be set to `true`. | `list(string)` | `null` | no | | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [cluster\_iam\_role\_name](#input\_cluster\_iam\_role\_name) | IAM role name for the cluster. If manage\_cluster\_iam\_resources is set to false, set this to reuse an existing IAM role. If manage\_cluster\_iam\_resources is set to true, set this to force the created role name. | `string` | `""` | no | diff --git a/variables.tf b/variables.tf index 8629362783..29e92afb32 100644 --- a/variables.tf +++ b/variables.tf @@ -248,13 +248,13 @@ variable "iam_path" { } variable "cluster_create_endpoint_private_access_sg_rule" { - description = "Whether to create security group rules for the access to the Amazon EKS private API server endpoint." + description = "Whether to create security group rules for the access to the Amazon EKS private API server endpoint. When is `true`, `cluster_endpoint_private_access_cidrs` must be setted." type = bool default = false } variable "cluster_endpoint_private_access_cidrs" { - description = "List of CIDR blocks which can access the Amazon EKS private API server endpoint." + description = "List of CIDR blocks which can access the Amazon EKS private API server endpoint. To use this `cluster_create_endpoint_private_access_sg_rule` must be set to `true`." type = list(string) default = null } From fcc2fdc9936e93c35d95fd279810787c6f6f62f8 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 10:35:51 +0200 Subject: [PATCH 326/962] feat: Define Root device name for Windows self-managed worker groups (#1401) NOTES: Since we now search only for Linux or Windows AMI if there is a worker groups for the corresponding plateform, we can now define different default root block device name for each plateform. Use locals `root_block_device_name` and `root_block_device_name_windows` to define your owns. --- local.tf | 21 +++++++++++---------- workers_launch_template.tf | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/local.tf b/local.tf index 11d16b7b87..09ff2de858 100644 --- a/local.tf +++ b/local.tf @@ -78,16 +78,17 @@ locals { warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. # Settings for launch templates - root_block_device_name = concat(data.aws_ami.eks_worker.*.root_device_name, [""])[0] # Root device name for workers. If non is provided, will assume default AMI was used. - root_kms_key_id = "" # The KMS key to use when encrypting the root storage device - launch_template_id = null # The id of the launch template used for managed node_groups - launch_template_version = "$Latest" # The latest version of the launch template to use in the autoscaling group - update_default_version = false # Update the autoscaling group launch template's default version upon each update - launch_template_placement_tenancy = "default" # The placement tenancy for instances - launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. - root_encrypted = false # Whether the volume should be encrypted or not - eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) - cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs + root_block_device_name = concat(data.aws_ami.eks_worker.*.root_device_name, [""])[0] # Root device name for Linux workers. If not provided, will assume default Linux AMI was used. + root_block_device_name_windows = concat(data.aws_ami.eks_worker_windows.*.root_device_name, [""])[0] # Root device name for Windows workers. If not provided, will assume default Windows AMI was used. + root_kms_key_id = "" # The KMS key to use when encrypting the root storage device + launch_template_id = null # The id of the launch template used for managed node_groups + launch_template_version = "$Latest" # The latest version of the launch template to use in the autoscaling group + update_default_version = false # Update the autoscaling group launch template's default version upon each update + launch_template_placement_tenancy = "default" # The placement tenancy for instances + launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. + root_encrypted = false # Whether the volume should be encrypted or not + eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) + cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs market_type = null metadata_http_endpoint = "enabled" # The state of the metadata service: enabled, disabled. metadata_http_tokens = "optional" # If session tokens are required: optional, required. diff --git a/workers_launch_template.tf b/workers_launch_template.tf index aaea9616cc..76225658ed 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -438,7 +438,7 @@ resource "aws_launch_template" "workers_launch_template" { device_name = lookup( var.worker_groups_launch_template[count.index], "root_block_device_name", - local.workers_group_defaults["root_block_device_name"], + lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? local.workers_group_defaults["root_block_device_name_windows"] : local.workers_group_defaults["root_block_device_name"], ) ebs { From 796cbead2fbe46952f2f40ac08f1cec5af2a7c84 Mon Sep 17 00:00:00 2001 From: Marc Haase <71422468+marc-slingshot@users.noreply.github.com> Date: Fri, 28 May 2021 05:56:02 -0700 Subject: [PATCH 327/962] feat: Add ability to use Security Groups as source for private endpoint access (#1274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGES: The private endpoint security group rule has been renamed to allow the use of CIDR blocks and Security Groups as source. This will delete the `cluster_private_access` Security Group Rule for existing cluster. Please rename by `aws_security_group_rule.cluster_private_access[0]` into `aws_security_group_rule.cluster_private_access_cidrs_source[0]`. Co-authored-by: Thierno IB. BARRY --- README.md | 7 +++++-- cluster.tf | 22 ++++++++++++++++++++++ variables.tf | 10 ++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7af8e066c2..cc29644c36 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,8 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [aws_security_group_rule.cluster_https_worker_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.cluster_primary_ingress_workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.cluster_private_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.cluster_private_access_cidrs_source](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.cluster_private_access_sg_source](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.workers_egress_internet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.workers_ingress_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.workers_ingress_cluster_https](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | @@ -228,8 +230,9 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | | [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster. See examples/secrets\_encryption/main.tf for example format |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | | [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | -| [cluster\_endpoint\_private\_access\_cidrs](#input\_cluster\_endpoint\_private\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS private API server endpoint. To use this `cluster_create_endpoint_private_access_sg_rule` must be set to `true`. | `list(string)` | `null` | no | -| [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | +| [cluster\_endpoint\_private\_access\_cidrs](#input\_cluster\_endpoint\_private\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS private API server endpoint. To use this `cluster_endpoint_private_access` and `cluster_create_endpoint_private_access_sg_rule` must be set to `true`. | `list(string)` | `null` | no | +| [cluster\_endpoint\_private\_access\_sg](#input\_cluster\_endpoint\_private\_access\_sg) | List of security group IDs which can access the Amazon EKS private API server endpoint. To use this `cluster_endpoint_private_access` and `cluster_create_endpoint_private_access_sg_rule` must be set to `true`. | `list(string)` | `null` | no | +| [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled. When it's set to `false` ensure to have a proper private access with `cluster_endpoint_private_access = true`. | `bool` | `true` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [cluster\_iam\_role\_name](#input\_cluster\_iam\_role\_name) | IAM role name for the cluster. If manage\_cluster\_iam\_resources is set to false, set this to reuse an existing IAM role. If manage\_cluster\_iam\_resources is set to true, set this to force the created role name. | `string` | `""` | no | | [cluster\_log\_kms\_key\_id](#input\_cluster\_log\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `""` | no | diff --git a/cluster.tf b/cluster.tf index 94581a4de3..cca7226d9e 100644 --- a/cluster.tf +++ b/cluster.tf @@ -87,6 +87,28 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { type = "ingress" } +resource "aws_security_group_rule" "cluster_private_access_cidrs_source" { + count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_cidrs != null ? 1 : 0 + type = "ingress" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = var.cluster_endpoint_private_access_cidrs + + security_group_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id +} + +resource "aws_security_group_rule" "cluster_private_access_sg_source" { + count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_sg != null ? length(var.cluster_endpoint_private_access_sg) : 0 + type = "ingress" + from_port = 443 + to_port = 443 + protocol = "tcp" + source_security_group_id = var.cluster_endpoint_private_access_sg[count.index] + + security_group_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id +} + resource "aws_security_group_rule" "cluster_private_access" { description = "Allow private K8S API ingress from custom source." count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access ? 1 : 0 diff --git a/variables.tf b/variables.tf index 29e92afb32..6cb9d20910 100644 --- a/variables.tf +++ b/variables.tf @@ -254,7 +254,13 @@ variable "cluster_create_endpoint_private_access_sg_rule" { } variable "cluster_endpoint_private_access_cidrs" { - description = "List of CIDR blocks which can access the Amazon EKS private API server endpoint. To use this `cluster_create_endpoint_private_access_sg_rule` must be set to `true`." + description = "List of CIDR blocks which can access the Amazon EKS private API server endpoint. To use this `cluster_endpoint_private_access` and `cluster_create_endpoint_private_access_sg_rule` must be set to `true`." + type = list(string) + default = null +} + +variable "cluster_endpoint_private_access_sg" { + description = "List of security group IDs which can access the Amazon EKS private API server endpoint. To use this `cluster_endpoint_private_access` and `cluster_create_endpoint_private_access_sg_rule` must be set to `true`." type = list(string) default = null } @@ -266,7 +272,7 @@ variable "cluster_endpoint_private_access" { } variable "cluster_endpoint_public_access" { - description = "Indicates whether or not the Amazon EKS public API server endpoint is enabled." + description = "Indicates whether or not the Amazon EKS public API server endpoint is enabled. When it's set to `false` ensure to have a proper private access with `cluster_endpoint_private_access = true`." type = bool default = true } From 1dc56fd5ab48481c922899d6903c2c786a7ce48b Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 15:03:38 +0200 Subject: [PATCH 328/962] chore: Update changelog (#1402) --- CHANGELOG.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b3071d153..5d9f57ad47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,36 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.0.0] - 2021-05-28 +FEATURES: +- Add ability to use Security Groups as source for private endpoint access ([#1274](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1274)) +- Define Root device name for Windows self-managed worker groups ([#1401](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1401)) +- Drop random pets from Managed Node Groups ([#1372](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1372)) +- Add multiple selectors on the creation of Fargate profile ([#1378](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1378)) +- Rename `config_output_path` into `kubeconfig_output_path` for naming consistency ([#1399](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1399)) +- Kubeconfig file should not be world or group readable by default ([#1114](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1114)) +- Add tags on network interfaces ([#1362](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1362)) +- Add instance store volume option for instances with local disk ([#1213](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1213)) + +BUG FIXES: +- Add back `depends_on` for `data.wait_for_cluster` ([#1389](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1389)) + +DOCS: +- Clarify about the `cluster_endpoint_private_access_cidrs` usage ([#1400](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1400)) +- Add KMS aliases handling to IAM permissions ([#1288](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1288)) + +BREAKING CHANGES: +- The private endpoint security group rule has been renamed to allow the use of CIDR blocks and Security Groups as source. This will delete the `cluster_private_access` Security Group Rule for existing cluster. Please rename by `aws_security_group_rule.cluster_private_access[0]` into `aws_security_group_rule.cluster_private_access_cidrs_source[0]`. +- We now decided to remove `random_pet` resources in Managed Node Groups (MNG). Those were used to recreate MNG if something change and also simulate the newly added argument `node_group_name_prefix`. But they were causing a lot of troubles. To upgrade the module without recreating your MNG, you will need to explicitly reuse their previous name and set them in your MNG `name` argument. Please see [upgrade docs](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/upgrades.md#upgrade-module-to-v1700-for-managed-node-groups) for more details. +- To support multiple selectors for Fargate profiles, we introduced the `selectors` argument which is a list of map. This will break previous configuration with a single selector `namespace` and `labels`. You'll need to rewrite your configuration to use the `selectors` argument. See [examples](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/examples/fargate/main.tf) dans [docs](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/modules/fargate/README.md) for details. +- The variable `config_output_path` is renamed into `kubeconfig_output_path` for naming consistency. Please upgrade your configuration accordingly. + +NOTES: +- Since we now search only for Linux or Windows AMI if there is a worker groups for the corresponding plateform, we can now define different default root block device name for each plateform. Use locals `root_block_device_name` and `root_block_device_name_windows` to define your owns. +- The kubeconfig file permission is not world and group readable anymore. The default permission is now `600`. This value can be changed with the variable `var.kubeconfig_file_permission`. + + ## [v16.2.0] - 2021-05-24 FEATURES: @@ -339,7 +369,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.2.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.0...HEAD +[v17.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.2.0...v17.0.0 [v16.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.1.0...v16.2.0 [v16.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.1...v16.1.0 [v16.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.0...v16.0.1 From 73c5107724ba1b403f45c3db9f0792f089aafd4c Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 15:31:41 +0200 Subject: [PATCH 329/962] fix: Default `root_volume_type` must be `gp2` (#1404) --- local.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local.tf b/local.tf index 09ff2de858..229937cd21 100644 --- a/local.tf +++ b/local.tf @@ -46,7 +46,7 @@ locals { spot_price = "" # Cost of spot instance. placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". root_volume_size = "100" # root volume size of workers instances. - root_volume_type = "gp3" # root volume type of workers instances, can be "standard", "gp3", "gp2", or "io1" + root_volume_type = "gp2" # root volume type of workers instances, can be "standard", "gp3", "gp2", or "io1" root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". root_volume_throughput = null # The amount of throughput to provision for a gp3 volume. key_name = "" # The key pair name that should be used for the instances in the autoscaling group From 3529caeeb88b29bc27e23e070b9faf02d69020af Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 15:33:43 +0200 Subject: [PATCH 330/962] chore: Update changelog (#1405) --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d9f57ad47..5d27d4bf0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.0.1] - 2021-05-28 +BUG FIXES: +- Default `root_volume_type` must be `gp2` ([#1404](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1404)) + + ## [v17.0.0] - 2021-05-28 FEATURES: @@ -369,7 +375,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.1...HEAD +[v17.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.0...v17.0.1 [v17.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.2.0...v17.0.0 [v16.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.1.0...v16.2.0 [v16.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.0.1...v16.1.0 From 65f5e2e4974fefaf0458212b5b0b086b11b4ff74 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 16:49:15 +0200 Subject: [PATCH 331/962] fix: Don't add tags on network interfaces because it's not supported yet in `terraform-provider-aws` (#1407) --- modules/node_groups/launch_template.tf | 14 -------------- workers_launch_template.tf | 18 ------------------ 2 files changed, 32 deletions(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 6ad2b68f85..bdd05b3a2c 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -99,20 +99,6 @@ resource "aws_launch_template" "workers" { ) } - # Supplying custom tags to EKS instances ENI's - tag_specifications { - resource_type = "network-interface" - - tags = merge( - var.tags, - lookup(var.node_groups_defaults, "additional_tags", {}), - lookup(var.node_groups[each.key], "additional_tags", {}), - { - Name = local.node_groups_names[each.key] - } - ) - } - # Tag the LT itself tags = merge( var.tags, diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 76225658ed..f7912ffe1b 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -563,24 +563,6 @@ resource "aws_launch_template" "workers_launch_template" { ) } - tag_specifications { - resource_type = "network-interface" - - tags = merge( - { - "Name" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}-eks_asg" - }, - { for tag_key, tag_value in var.tags : - tag_key => tag_value - if tag_key != "Name" && !contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) - } - ) - } - tags = var.tags lifecycle { From 2df401fe7f203092221d94ac631f0d469995c6b5 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 16:52:54 +0200 Subject: [PATCH 332/962] chore: Update changelog (#1408) --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d27d4bf0e..44544bd77a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.0.2] - 2021-05-28 +BUG FIXES: +- Don't add tags on network interfaces because it's not supported yet in `terraform-provider-aws` ([#1407](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1407)) + + ## [v17.0.1] - 2021-05-28 BUG FIXES: @@ -375,7 +381,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.1...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.2...HEAD +[v17.0.2]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.1...v17.0.2 [v17.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.0...v17.0.1 [v17.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.2.0...v17.0.0 [v16.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.1.0...v16.2.0 From 1d848b56bf7c43ce708bd7cdee45589f816e05f0 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 21:11:02 +0200 Subject: [PATCH 333/962] fix: Remove duplicated security group rule for EKS private access endpoint (#1412) NOTES: In this bug fix, we remove a duplicated security rule introduced during a merge conflict resolution in [#1274](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/1274) --- README.md | 1 - cluster.tf | 14 ++------------ data.tf | 3 ++- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index cc29644c36..607c8be86c 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,6 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [aws_security_group_rule.cluster_egress_internet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.cluster_https_worker_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.cluster_primary_ingress_workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.cluster_private_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.cluster_private_access_cidrs_source](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.cluster_private_access_sg_source](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.workers_egress_internet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | diff --git a/cluster.tf b/cluster.tf index cca7226d9e..13d38a09a7 100644 --- a/cluster.tf +++ b/cluster.tf @@ -89,6 +89,7 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { resource "aws_security_group_rule" "cluster_private_access_cidrs_source" { count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_cidrs != null ? 1 : 0 + description = "Allow private K8S API ingress from custom CIDR source." type = "ingress" from_port = 443 to_port = 443 @@ -100,6 +101,7 @@ resource "aws_security_group_rule" "cluster_private_access_cidrs_source" { resource "aws_security_group_rule" "cluster_private_access_sg_source" { count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_sg != null ? length(var.cluster_endpoint_private_access_sg) : 0 + description = "Allow private K8S API ingress from custom Security Groups source." type = "ingress" from_port = 443 to_port = 443 @@ -109,18 +111,6 @@ resource "aws_security_group_rule" "cluster_private_access_sg_source" { security_group_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id } -resource "aws_security_group_rule" "cluster_private_access" { - description = "Allow private K8S API ingress from custom source." - count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access ? 1 : 0 - type = "ingress" - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = var.cluster_endpoint_private_access_cidrs - - security_group_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id -} - resource "aws_iam_role" "cluster" { count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 name_prefix = var.cluster_iam_role_name != "" ? null : var.cluster_name diff --git a/data.tf b/data.tf index 875ca5e8ee..f97692e53e 100644 --- a/data.tf +++ b/data.tf @@ -94,6 +94,7 @@ data "http" "wait_for_cluster" { depends_on = [ aws_eks_cluster.this, - aws_security_group_rule.cluster_private_access, + aws_security_group_rule.cluster_private_access_sg_source, + aws_security_group_rule.cluster_private_access_cidrs_source, ] } From f993c3622a02e0180e34b767f1c3d4714a27fba4 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 22:12:29 +0200 Subject: [PATCH 334/962] fix: Fix AMI filtering when the default platform is provided in `var.workers_group_defaults` (#1413) --- local.tf | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/local.tf b/local.tf index 229937cd21..3da2674c74 100644 --- a/local.tf +++ b/local.tf @@ -6,6 +6,7 @@ locals { cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id + default_platform = "linux" default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] default_ami_id_linux = local.workers_group_defaults.ami_id != "" ? local.workers_group_defaults.ami_id : concat(data.aws_ami.eks_worker.*.id, [""])[0] default_ami_id_windows = local.workers_group_defaults.ami_id_windows != "" ? local.workers_group_defaults.ami_id_windows : concat(data.aws_ami.eks_worker_windows.*.id, [""])[0] @@ -15,8 +16,26 @@ locals { worker_group_count = length(var.worker_groups) worker_group_launch_template_count = length(var.worker_groups_launch_template) - worker_has_linux_ami = length([for x in concat(var.worker_groups, var.worker_groups_launch_template) : x if lookup(x, "platform", "linux") == "linux"]) > 0 - worker_has_windows_ami = length([for x in concat(var.worker_groups, var.worker_groups_launch_template) : x if lookup(x, "platform", "linux") == "windows"]) > 0 + worker_has_linux_ami = length([for x in concat(var.worker_groups, var.worker_groups_launch_template) : x if lookup( + x, + "platform", + # Fallback on default `platform` if it's not defined in current worker group + lookup( + merge({ platform = local.default_platform }, var.workers_group_defaults), + "platform", + null + ) + ) == "linux"]) > 0 + worker_has_windows_ami = length([for x in concat(var.worker_groups, var.worker_groups_launch_template) : x if lookup( + x, + "platform", + # Fallback on default `platform` if it's not defined in current worker group + lookup( + merge({ platform = local.default_platform }, var.workers_group_defaults), + "platform", + null + ) + ) == "windows"]) > 0 worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" # Windows nodes are available from k8s 1.14. If cluster version is less than 1.14, fix ami filter to some constant to not fail on 'terraform plan'. @@ -72,7 +91,7 @@ locals { placement_group = null # The name of the placement group into which to launch the instances, if any. service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. - platform = "linux" # Platform of workers. either "linux" or "windows" + platform = local.default_platform # Platform of workers. Either "linux" or "windows". additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults additional_instance_store_volumes = [] # A list of additional instance store (local disk) volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), virtual_name. warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. From 6c1eff7c109290ac93eae009d3a2e8b2be1075e9 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 22:14:58 +0200 Subject: [PATCH 335/962] chore: Update changelog (#1414) --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44544bd77a..4dda0ee0a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,16 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.0.3] - 2021-05-28 +BUG FIXES: +- Fix AMI filtering when the default platform is provided in `var.workers_group_defaults` ([#1413](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1413)) +- Remove duplicated security group rule for EKS private access endpoint ([#1412](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1412)) + +NOTES: +- In this bug fix, we remove a duplicated security rule introduced during a merge conflict resolution in [[#1274](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1274)](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/1274) + + ## [v17.0.2] - 2021-05-28 BUG FIXES: @@ -381,7 +391,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.2...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.3...HEAD +[v17.0.3]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.2...v17.0.3 [v17.0.2]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.1...v17.0.2 [v17.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.0...v17.0.1 [v17.0.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v16.2.0...v17.0.0 From 6ad44056e3e18aacb64cbdfe8f3de0b6ec0c595b Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 28 May 2021 22:22:40 +0200 Subject: [PATCH 336/962] chore: Fix typo in upgrades docs --- docs/upgrades.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrades.md b/docs/upgrades.md index 66da5b7b75..88d29ae1bf 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -57,4 +57,4 @@ Plan: 0 to add, 0 to change, 1 to destroy. ``` 5. If everything sounds good to you, run `terraform apply` -After the first apply, we recommand you to create a new node group and let the module use the `node_group_name_prefix` (by removing the `name` argument) to generate names and collision during node groups re-creation if needed, because the lifce cycle is `create_before_destroy = true`. +After the first apply, we recommand you to create a new node group and let the module use the `node_group_name_prefix` (by removing the `name` argument) to generate names and avoid collision during node groups re-creation if needed, because the lifce cycle is `create_before_destroy = true`. From 686193ed4eb52b66771f5523bfddfa5681cc45a5 Mon Sep 17 00:00:00 2001 From: SNA <66044256+SNA-rh@users.noreply.github.com> Date: Tue, 1 Jun 2021 16:10:31 -0500 Subject: [PATCH 337/962] feat: Add `var.wait_for_cluster_timeout` to allow configuring the wait for cluster timeout (#1420) --- README.md | 1 + data.tf | 2 +- variables.tf | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 607c8be86c..45b06aaeba 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [subnets](#input\_subnets) | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | [tags](#input\_tags) | A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | | [vpc\_id](#input\_vpc\_id) | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | +| [wait\_for\_cluster\_timeout](#wait\_for\_cluster\_timeout) | Allows for a configurable timeout (in seconds) when waiting for a cluster to come up | `number` | `300` | no | | [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | [worker\_ami\_name\_filter](#input\_worker\_ami\_name\_filter) | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | | [worker\_ami\_name\_filter\_windows](#input\_worker\_ami\_name\_filter\_windows) | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | diff --git a/data.tf b/data.tf index f97692e53e..bc80e74a69 100644 --- a/data.tf +++ b/data.tf @@ -90,7 +90,7 @@ data "http" "wait_for_cluster" { count = var.create_eks && var.manage_aws_auth ? 1 : 0 url = format("%s/healthz", aws_eks_cluster.this[0].endpoint) ca_certificate = base64decode(coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0]) - timeout = 300 + timeout = var.wait_for_cluster_timeout depends_on = [ aws_eks_cluster.this, diff --git a/variables.tf b/variables.tf index 6cb9d20910..b7d560e402 100644 --- a/variables.tf +++ b/variables.tf @@ -387,3 +387,9 @@ variable "workers_egress_cidrs" { type = list(string) default = ["0.0.0.0/0"] } + +variable "wait_for_cluster_timeout" { + description = "A timeout (in seconds) to wait for cluster to be available." + type = number + default = 300 +} From bf4fd83a854870d7e02aa184d9f9b4fdb3632a69 Mon Sep 17 00:00:00 2001 From: Patrick Moelk Date: Wed, 2 Jun 2021 15:03:40 +0200 Subject: [PATCH 338/962] docs: Add `iam:TagOpenIDConnectProvider` to list of necessary permissions (#1423) --- docs/iam-permissions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index 27f88f22e1..b5810d9435 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -124,6 +124,7 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "iam:PassRole", "iam:PutRolePolicy", "iam:RemoveRoleFromInstanceProfile", + "iam:TagOpenIDConnectProvider", "iam:TagRole", "iam:UntagRole", "iam:UpdateAssumeRolePolicy", From 1e2c32430f458409771938c16a3dc437cd657d02 Mon Sep 17 00:00:00 2001 From: LAKostis Date: Thu, 3 Jun 2021 23:41:39 +0200 Subject: [PATCH 339/962] feat: Add `capacity_rebalance` support for self-managed worker groups (#1326) --- local.tf | 1 + workers.tf | 5 +++++ workers_launch_template.tf | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/local.tf b/local.tf index 3da2674c74..dc836a4729 100644 --- a/local.tf +++ b/local.tf @@ -127,6 +127,7 @@ locals { instance_refresh_min_healthy_percentage = 90 # The amount of capacity in the ASG that must remain healthy during an instance refresh, as a percentage of the ASG's desired capacity. instance_refresh_instance_warmup = null # The number of seconds until a newly launched instance is configured and ready to use. Defaults to the ASG's health check grace period. instance_refresh_triggers = [] # Set of additional property names that will trigger an Instance Refresh. A refresh will always be triggered by a change in any of launch_configuration, launch_template, or mixed_instances_policy. + capacity_rebalance = false # Enable capacity rebalance } workers_group_defaults = merge( diff --git a/workers.tf b/workers.tf index 897805df98..31a2ffd568 100644 --- a/workers.tf +++ b/workers.tf @@ -97,6 +97,11 @@ resource "aws_autoscaling_group" "workers" { "health_check_grace_period", local.workers_group_defaults["health_check_grace_period"] ) + capacity_rebalance = lookup( + var.worker_groups[count.index], + "capacity_rebalance", + local.workers_group_defaults["capacity_rebalance"] + ) dynamic "initial_lifecycle_hook" { for_each = var.worker_create_initial_lifecycle_hooks ? lookup(var.worker_groups[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) : [] diff --git a/workers_launch_template.tf b/workers_launch_template.tf index f7912ffe1b..6e14b7dcb0 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -96,6 +96,11 @@ resource "aws_autoscaling_group" "workers_launch_template" { "health_check_grace_period", local.workers_group_defaults["health_check_grace_period"] ) + capacity_rebalance = lookup( + var.worker_groups_launch_template[count.index], + "capacity_rebalance", + local.workers_group_defaults["capacity_rebalance"] + ) dynamic "mixed_instances_policy" { iterator = item From 4c1f272e2993d5cef1136d7c32d413eeae05b367 Mon Sep 17 00:00:00 2001 From: Kevin Lefevre Date: Fri, 4 Jun 2021 00:07:03 +0200 Subject: [PATCH 340/962] feat: Allow to choose launch template version for Managed Node Groups when `create_launch_template` is set to `true` (#1419) --- modules/node_groups/node_groups.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index a0ae456fa7..35eb058fb0 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -48,7 +48,7 @@ resource "aws_eks_node_group" "workers" { dynamic "launch_template" { for_each = each.value["launch_template_id"] == null && each.value["create_launch_template"] ? [{ id = aws_launch_template.workers[each.key].id - version = aws_launch_template.workers[each.key].latest_version + version = each.value["launch_template_version"] }] : [] content { From e6d76d0a069c0192bdace25e992e54709b667e5c Mon Sep 17 00:00:00 2001 From: Xiangxuan Liu <9570112+nauxliu@users.noreply.github.com> Date: Fri, 4 Jun 2021 06:11:15 +0800 Subject: [PATCH 341/962] feat: Add support for Managed Node Groups (`node_groups`) taints (#1424) --- examples/managed_node_groups/main.tf | 7 +++++++ modules/node_groups/README.md | 5 +++-- modules/node_groups/locals.tf | 1 + modules/node_groups/node_groups.tf | 10 ++++++++++ modules/node_groups/versions.tf | 2 +- 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index dc799e4203..4c8b3fed6c 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -88,6 +88,13 @@ module "eks" { additional_tags = { ExtraTag = "example" } + taints = [ + { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + ] } } diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index d9bb496a83..65fde29883 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -44,6 +44,7 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | source\_security\_group\_ids | Source security groups for remote access to workers | list(string) | If key\_name is specified: THE REMOTE ACCESS WILL BE OPENED TO THE WORLD | | subnets | Subnets to contain workers | list(string) | `var.workers_group_defaults[subnets]` | | version | Kubernetes version | string | Provider default behavior | +| taints | Kubernetes node taints | list(map) | empty | ## Requirements @@ -51,13 +52,13 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.40.0 | +| [aws](#requirement\_aws) | >= 3.43.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.40.0 | +| [aws](#provider\_aws) | >= 3.43.0 | | [cloudinit](#provider\_cloudinit) | n/a | ## Modules diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 192b7a01d1..ee026c86d7 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -21,6 +21,7 @@ locals { public_ip = var.workers_group_defaults["public_ip"] pre_userdata = var.workers_group_defaults["pre_userdata"] additional_security_group_ids = var.workers_group_defaults["additional_security_group_ids"] + taints = [] }, var.node_groups_defaults, v, diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 35eb058fb0..6a1e3938e3 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -57,6 +57,16 @@ resource "aws_eks_node_group" "workers" { } } + dynamic "taint" { + for_each = each.value["taints"] + + content { + key = taint.value["key"] + value = taint.value["value"] + effect = taint.value["effect"] + } + } + version = lookup(each.value, "version", null) labels = merge( diff --git a/modules/node_groups/versions.tf b/modules/node_groups/versions.tf index 2051547e51..c68eb70217 100644 --- a/modules/node_groups/versions.tf +++ b/modules/node_groups/versions.tf @@ -2,6 +2,6 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.40.0" + aws = ">= 3.43.0" } } From 9022013844a61193a2f8764311fb679747807f5c Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Wed, 9 Jun 2021 20:02:28 +0200 Subject: [PATCH 342/962] chore: Update changelog (#1435) --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dda0ee0a8..305584782a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.1.0] - 2021-06-04 +FEATURES: +- Add support for Managed Node Groups (`node_groups`) taints ([#1424](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1424)) +- Allow to choose launch template version for Managed Node Groups when `create_launch_template` is set to `true` ([#1419](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1419)) +- Add `capacity_rebalance` support for self-managed worker groups ([#1326](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1326)) +- Add `var.wait_for_cluster_timeout` to allow configuring the wait for cluster timeout ([#1420](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1420)) + + ## [v17.0.3] - 2021-05-28 BUG FIXES: @@ -391,7 +400,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.3...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.1.0...HEAD +[v17.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.3...v17.1.0 [v17.0.3]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.2...v17.0.3 [v17.0.2]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.1...v17.0.2 [v17.0.1]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.0...v17.0.1 From 3b1229837ad61d6e3420196fe3870cc406940cc8 Mon Sep 17 00:00:00 2001 From: Ryan White <4404175+alzabo@users.noreply.github.com> Date: Wed, 25 Aug 2021 09:21:08 -0400 Subject: [PATCH 343/962] feat: Enable ebs_optimized setting for node_groups (#1459) --- .pre-commit-config.yaml | 2 +- README.md | 2 +- examples/launch_templates/main.tf | 1 + modules/node_groups/README.md | 2 ++ modules/node_groups/launch_template.tf | 2 ++ modules/node_groups/locals.tf | 1 + modules/node_groups/variables.tf | 6 ++++++ node_groups.tf | 1 + 8 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ef8a79bb9..6396765535 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.48.0 + rev: v1.50.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index 45b06aaeba..6f680822b1 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [subnets](#input\_subnets) | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | [tags](#input\_tags) | A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | | [vpc\_id](#input\_vpc\_id) | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | -| [wait\_for\_cluster\_timeout](#wait\_for\_cluster\_timeout) | Allows for a configurable timeout (in seconds) when waiting for a cluster to come up | `number` | `300` | no | +| [wait\_for\_cluster\_timeout](#input\_wait\_for\_cluster\_timeout) | A timeout (in seconds) to wait for cluster to be available. | `number` | `300` | no | | [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | [worker\_ami\_name\_filter](#input\_worker\_ami\_name\_filter) | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | | [worker\_ami\_name\_filter\_windows](#input\_worker\_ami\_name\_filter\_windows) | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index fe5df294ce..c01256fe58 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -59,6 +59,7 @@ module "eks" { instance_type = "t3.medium" asg_desired_capacity = 1 public_ip = true + ebs_optimized = true }, { name = "worker-group-3" diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 65fde29883..ab7bb512dd 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -25,6 +25,7 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` | | disk\_size | Workers' disk size | number | Provider default behavior | | disk\_type | Workers' disk type. Require `create_launch_template` to be `true`| number | `gp3` | +| ebs\_optimized | Enables/disables EBS optimization. Require `create_launch_template` to be `true` | bool | `true` if defined `instance\_types` are not present in `var.ebs\_optimized\_not\_supported` | | enable_monitoring | Enables/disables detailed monitoring. Require `create_launch_template` to be `true`| bool | `true` | | eni_delete | Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) | bool | `true` | | force\_update\_version | Force version update if existing pods are unable to be drained due to a pod disruption budget issue. | bool | Provider default behavior | @@ -80,6 +81,7 @@ No modules. | [cluster\_name](#input\_cluster\_name) | Name of parent cluster | `string` | n/a | yes | | [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | [default\_iam\_role\_arn](#input\_default\_iam\_role\_arn) | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | n/a | yes | +| [ebs\_optimized\_not\_supported](#input\_ebs\_optimized\_not\_supported) | List of instance types that do not support EBS optimization | `list(string)` | `[]` | no | | [ng\_depends\_on](#input\_ng\_depends\_on) | List of references to other resources this submodule depends on | `any` | `null` | no | | [node\_groups](#input\_node\_groups) | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | | [node\_groups\_defaults](#input\_node\_groups\_defaults) | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | n/a | yes | diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index bdd05b3a2c..eab50b6322 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -39,6 +39,8 @@ resource "aws_launch_template" "workers" { } } + ebs_optimized = lookup(each.value, "ebs_optimized", !contains(var.ebs_optimized_not_supported, element(each.value.instance_types, 0))) + instance_type = each.value["set_instance_types_on_lt"] ? element(each.value.instance_types, 0) : null monitoring { diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index ee026c86d7..89dcd844c4 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -22,6 +22,7 @@ locals { pre_userdata = var.workers_group_defaults["pre_userdata"] additional_security_group_ids = var.workers_group_defaults["additional_security_group_ids"] taints = [] + ebs_optimized = null }, var.node_groups_defaults, v, diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf index 585beb5f91..52209e5ef0 100644 --- a/modules/node_groups/variables.tf +++ b/modules/node_groups/variables.tf @@ -54,3 +54,9 @@ variable "ng_depends_on" { type = any default = null } + +variable "ebs_optimized_not_supported" { + description = "List of instance types that do not support EBS optimization" + type = list(string) + default = [] +} diff --git a/node_groups.tf b/node_groups.tf index d98979310f..ec483b8f02 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -9,6 +9,7 @@ module "node_groups" { tags = var.tags node_groups_defaults = var.node_groups_defaults node_groups = var.node_groups + ebs_optimized_not_supported = local.ebs_optimized_not_supported # Hack to ensure ordering of resource creation. # This is a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 From 6067290c0e979993af10309f0bbacbe16441e42e Mon Sep 17 00:00:00 2001 From: Ryan White <4404175+alzabo@users.noreply.github.com> Date: Wed, 25 Aug 2021 09:22:53 -0400 Subject: [PATCH 344/962] feat: Support for encrypted root disk in node_groups (#1428) --- modules/node_groups/README.md | 2 ++ modules/node_groups/launch_template.tf | 2 ++ modules/node_groups/locals.tf | 2 ++ 3 files changed, 6 insertions(+) diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index ab7bb512dd..448b9f7f3c 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -23,6 +23,8 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | capacity\_type | Type of instance capacity to provision. Options are `ON_DEMAND` and `SPOT` | string | Provider default behavior | | create_launch_template | Create and use a default launch template | bool | `false` | | desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` | +| disk\_encrypted | Whether the root disk will be encrypyted. Requires `create_launch_template` to be `true` and `disk_kms_key_id` to be set | bool | false | +| disk\_kms\_key\_id | KMS Key used to encrypt the root disk. Requires both `create_launch_template` and `disk_encrypted` to be `true` | string | "" | | disk\_size | Workers' disk size | number | Provider default behavior | | disk\_type | Workers' disk type. Require `create_launch_template` to be `true`| number | `gp3` | | ebs\_optimized | Enables/disables EBS optimization. Require `create_launch_template` to be `true` | bool | `true` if defined `instance\_types` are not present in `var.ebs\_optimized\_not\_supported` | diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index eab50b6322..a528a3dc49 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -35,6 +35,8 @@ resource "aws_launch_template" "workers" { ebs { volume_size = lookup(each.value, "disk_size", null) volume_type = lookup(each.value, "disk_type", null) + encrypted = lookup(each.value, "disk_encrypted", null) + kms_key_id = lookup(each.value, "disk_kms_key_id", null) delete_on_termination = true } } diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 89dcd844c4..945a7e325c 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -16,6 +16,8 @@ locals { kubelet_extra_args = var.workers_group_defaults["kubelet_extra_args"] disk_size = var.workers_group_defaults["root_volume_size"] disk_type = var.workers_group_defaults["root_volume_type"] + disk_encrypted = var.workers_group_defaults["root_encrypted"] + disk_kms_key_id = var.workers_group_defaults["root_kms_key_id"] enable_monitoring = var.workers_group_defaults["enable_monitoring"] eni_delete = var.workers_group_defaults["eni_delete"] public_ip = var.workers_group_defaults["public_ip"] From b15d8998e65d4b81fbd0d2206092b2d6060371a4 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Wed, 25 Aug 2021 15:24:57 +0200 Subject: [PATCH 345/962] Updated CHANGELOG --- CHANGELOG.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 305584782a..f6515c4e56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,15 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.2.0] - 2021-08-25 +FEATURES: +- Support for encrypted root disk in node_groups ([#1428](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1428)) +- Enable ebs_optimized setting for node_groups ([#1459](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1459)) + + -## [v17.1.0] - 2021-06-04 +## [v17.1.0] - 2021-06-09 FEATURES: - Add support for Managed Node Groups (`node_groups`) taints ([#1424](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1424)) - Allow to choose launch template version for Managed Node Groups when `create_launch_template` is set to `true` ([#1419](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1419)) @@ -400,7 +407,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.1.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.2.0...HEAD +[v17.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.1.0...v17.2.0 [v17.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.3...v17.1.0 [v17.0.3]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.2...v17.0.3 [v17.0.2]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.1...v17.0.2 From 4f683ab776419947c0221e7ebd8525a37d1978b6 Mon Sep 17 00:00:00 2001 From: Kevin Lefevre Date: Wed, 25 Aug 2021 16:42:29 +0200 Subject: [PATCH 346/962] chore: extract only terraform-docs binary during GH Actions run (#1541) --- .github/workflows/pre-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index ab9aef239c..f5d5d776a8 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -91,7 +91,7 @@ jobs: - name: Install pre-commit dependencies run: | pip install pre-commit - curl -Lo ./terraform-docs.tar.gz https://github.com/terraform-docs/terraform-docs/releases/download/v0.13.0/terraform-docs-v0.13.0-$(uname)-amd64.tar.gz && tar -xzf terraform-docs.tar.gz && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ + curl -Lo ./terraform-docs.tar.gz https://github.com/terraform-docs/terraform-docs/releases/download/v0.13.0/terraform-docs-v0.13.0-$(uname)-amd64.tar.gz && tar -xzf terraform-docs.tar.gz terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/ - name: Execute pre-commit # Run all pre-commit checks on max version supported From d17007b542b10c652914a17f5cb9a9ea8c1d86ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Hidalgo=20Garc=C3=ADa?= Date: Wed, 25 Aug 2021 16:54:29 +0200 Subject: [PATCH 347/962] fix: Fixed launch template version infinite plan issue and improved rolling updates (#1447) --- modules/node_groups/README.md | 1 + modules/node_groups/launch_template.tf | 2 +- modules/node_groups/locals.tf | 1 + modules/node_groups/node_groups.tf | 6 ++++-- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 448b9f7f3c..1ac6612eab 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -48,6 +48,7 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | subnets | Subnets to contain workers | list(string) | `var.workers_group_defaults[subnets]` | | version | Kubernetes version | string | Provider default behavior | | taints | Kubernetes node taints | list(map) | empty | +| update_default_version | Whether or not to set the new launch template version the Default | bool | `true` | ## Requirements diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index a528a3dc49..12d7dc3fc4 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -27,7 +27,7 @@ resource "aws_launch_template" "workers" { name_prefix = local.node_groups_names[each.key] description = format("EKS Managed Node Group custom LT for %s", local.node_groups_names[each.key]) - update_default_version = true + update_default_version = lookup(each.value, "update_default_version", true) block_device_mappings { device_name = "/dev/xvda" diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 945a7e325c..da8d20e582 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -24,6 +24,7 @@ locals { pre_userdata = var.workers_group_defaults["pre_userdata"] additional_security_group_ids = var.workers_group_defaults["additional_security_group_ids"] taints = [] + update_default_version = true ebs_optimized = null }, var.node_groups_defaults, diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 6a1e3938e3..4e3bf596ad 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -47,8 +47,10 @@ resource "aws_eks_node_group" "workers" { dynamic "launch_template" { for_each = each.value["launch_template_id"] == null && each.value["create_launch_template"] ? [{ - id = aws_launch_template.workers[each.key].id - version = each.value["launch_template_version"] + id = aws_launch_template.workers[each.key].id + version = each.value["launch_template_version"] == "$Latest" ? aws_launch_template.workers[each.key].latest_version : ( + each.value["launch_template_version"] == "$Default" ? aws_launch_template.workers[each.key].default_version : each.value["launch_template_version"] + ) }] : [] content { From 4e5b8bf095e417c83cfaa520c30712cad1e316bf Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Wed, 25 Aug 2021 16:55:56 +0200 Subject: [PATCH 348/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6515c4e56..3ba032b8a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.3.0] - 2021-08-25 +BUG FIXES: +- Fixed launch template version infinite plan issue and improved rolling updates ([#1447](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1447)) + + ## [v17.2.0] - 2021-08-25 FEATURES: @@ -407,7 +413,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.2.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.3.0...HEAD +[v17.3.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.2.0...v17.3.0 [v17.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.1.0...v17.2.0 [v17.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.3...v17.1.0 [v17.0.3]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.2...v17.0.3 From cd01601805153c3985b16ab647c56ce8cf5e58c8 Mon Sep 17 00:00:00 2001 From: Mikhail Naletov <36846182+okgolove@users.noreply.github.com> Date: Wed, 25 Aug 2021 19:13:07 +0300 Subject: [PATCH 349/962] chore: "Desirec" misspelling fix (#1450) --- local.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local.tf b/local.tf index dc836a4729..9d2d7fdd86 100644 --- a/local.tf +++ b/local.tf @@ -52,7 +52,7 @@ locals { tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. ami_id = "" # AMI ID for the eks linux based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. ami_id_windows = "" # AMI ID for the eks windows based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. - asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. + asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desired_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. asg_max_size = "3" # Maximum worker capacity in the autoscaling group. asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. asg_force_delete = false # Enable forced deletion for the autoscaling group. From 880b08cb6a6158b34e488645cbf2781cbb9863a4 Mon Sep 17 00:00:00 2001 From: Pascal Bourdier Date: Wed, 25 Aug 2021 18:19:10 +0200 Subject: [PATCH 350/962] docs: Updated autoscaling.md (#1515) --- docs/autoscaling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/autoscaling.md b/docs/autoscaling.md index c5ba615433..3c1aa5ee93 100644 --- a/docs/autoscaling.md +++ b/docs/autoscaling.md @@ -13,7 +13,7 @@ If you don't want to use IRSA then you will need to attach the IAM policy to the ```hcl resource "aws_iam_role_policy_attachment" "workers_autoscaling" { policy_arn = aws_iam_policy.worker_autoscaling.arn - role = module.my_cluster.worker_iam_role_name[0] + role = module.my_cluster.worker_iam_role_name } resource "aws_iam_policy" "worker_autoscaling" { From 904ad94fe27878885e000ff4717fc5e3b7d7b438 Mon Sep 17 00:00:00 2001 From: Andrew Conti Date: Wed, 25 Aug 2021 14:42:37 -0400 Subject: [PATCH 351/962] docs: Add missing comma to docs/iam-permissions.md (#1437) --- docs/iam-permissions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index b5810d9435..7c6d19c26e 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -109,7 +109,7 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "iam:DeleteInstanceProfile", "iam:DeleteOpenIDConnectProvider", "iam:DeletePolicy", - "iam:DeletePolicyVersion" + "iam:DeletePolicyVersion", "iam:DeleteRole", "iam:DeleteRolePolicy", "iam:DeleteServiceLinkedRole", From cb0e67736415dd0d8c452b6228432d8e2052e3dc Mon Sep 17 00:00:00 2001 From: Aidan Mundy Date: Thu, 26 Aug 2021 04:16:23 -0400 Subject: [PATCH 352/962] fix: Allow instance `Name` tag to be overwritten (#1538) --- modules/node_groups/launch_template.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 12d7dc3fc4..f70d2f5838 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -81,11 +81,11 @@ resource "aws_launch_template" "workers" { tags = merge( var.tags, - lookup(var.node_groups_defaults, "additional_tags", {}), - lookup(var.node_groups[each.key], "additional_tags", {}), { Name = local.node_groups_names[each.key] - } + }, + lookup(var.node_groups_defaults, "additional_tags", {}), + lookup(var.node_groups[each.key], "additional_tags", {}) ) } @@ -95,11 +95,11 @@ resource "aws_launch_template" "workers" { tags = merge( var.tags, - lookup(var.node_groups_defaults, "additional_tags", {}), - lookup(var.node_groups[each.key], "additional_tags", {}), { Name = local.node_groups_names[each.key] - } + }, + lookup(var.node_groups_defaults, "additional_tags", {}), + lookup(var.node_groups[each.key], "additional_tags", {}) ) } From 551c6d63bb02cb097c3bc7ae1988fadec4c392c5 Mon Sep 17 00:00:00 2001 From: Phileas Lebada Date: Thu, 26 Aug 2021 13:02:16 +0200 Subject: [PATCH 353/962] fix: Discourage usage of iam_policy_attachment in example (#1529) --- examples/bottlerocket/main.tf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/bottlerocket/main.tf b/examples/bottlerocket/main.tf index 86166df0e5..b254af0a12 100644 --- a/examples/bottlerocket/main.tf +++ b/examples/bottlerocket/main.tf @@ -57,8 +57,7 @@ EOT # SSM policy for bottlerocket control container access # https://github.com/bottlerocket-os/bottlerocket/blob/develop/QUICKSTART-EKS.md#enabling-ssm -resource "aws_iam_policy_attachment" "ssm" { - name = "ssm" - roles = [module.eks.worker_iam_role_name] +resource "aws_iam_role_policy_attachment" "ssm" { + role = module.eks.worker_iam_role_name policy_arn = data.aws_iam_policy.ssm.arn } From e3216e3cf80cb59089ba0e0365c6650520000aaf Mon Sep 17 00:00:00 2001 From: Br0mZ <38252710+Br0mZ@users.noreply.github.com> Date: Thu, 26 Aug 2021 13:04:35 +0200 Subject: [PATCH 354/962] docs: Fix cluster-autoscaler tags in irsa example (#1436) --- examples/irsa/README.md | 2 +- examples/irsa/irsa.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/irsa/README.md b/examples/irsa/README.md index 53dc500dab..ab81f649d7 100644 --- a/examples/irsa/README.md +++ b/examples/irsa/README.md @@ -1,6 +1,6 @@ # IAM Roles for Service Accounts -This example shows how to create an IAM role to be used for a Kubernetes `ServiceAccount`. It will create a policy and role to be used by the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) using the [public Helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler-chart). +This example shows how to create an IAM role to be used for a Kubernetes `ServiceAccount`. It will create a policy and role to be used by the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) using the [public Helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler). The AWS documentation for IRSA is here: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html diff --git a/examples/irsa/irsa.tf b/examples/irsa/irsa.tf index 1775e33ecb..c841d4f266 100644 --- a/examples/irsa/irsa.tf +++ b/examples/irsa/irsa.tf @@ -44,7 +44,7 @@ data "aws_iam_policy_document" "cluster_autoscaler" { condition { test = "StringEquals" - variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${module.eks.cluster_id}" + variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/${module.eks.cluster_id}" values = ["owned"] } From 4be3cc30458a83dbd6d3d00dd184103de2c15542 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Fri, 27 Aug 2021 09:37:37 +0200 Subject: [PATCH 355/962] Updated CHANGELOG --- CHANGELOG.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ba032b8a3..fd9c24d874 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,18 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.4.0] - 2021-08-26 +BUG FIXES: +- Discourage usage of iam_policy_attachment in example ([#1529](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1529)) +- Allow instance `Name` tag to be overwritten ([#1538](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1538)) + +DOCS: +- Fix cluster-autoscaler tags in irsa example ([#1436](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1436)) +- Add missing comma to docs/iam-permissions.md ([#1437](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1437)) +- Updated autoscaling.md ([#1515](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1515)) + + ## [v17.3.0] - 2021-08-25 BUG FIXES: @@ -413,7 +425,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.3.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.4.0...HEAD +[v17.4.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.3.0...v17.4.0 [v17.3.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.2.0...v17.3.0 [v17.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.1.0...v17.2.0 [v17.1.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.0.3...v17.1.0 From 6fb02c4fc430acbde444da0ac93fcc9e55f1cc03 Mon Sep 17 00:00:00 2001 From: Scott Cabrinha Date: Tue, 31 Aug 2021 03:27:04 -0700 Subject: [PATCH 356/962] feat: Allow users to add more Audiences to OpenID Connect (#1451) --- README.md | 1 + irsa.tf | 2 +- local.tf | 2 +- variables.tf | 6 ++++++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6f680822b1..621202c1fe 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [map\_users](#input\_map\_users) | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | | [node\_groups](#input\_node\_groups) | Map of map of node groups to create. See `node_groups` module's documentation for more details | `any` | `{}` | no | | [node\_groups\_defaults](#input\_node\_groups\_defaults) | Map of values to be applied to all node groups. See `node_groups` module's documentation for more details | `any` | `{}` | no | +| [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider. | `list(string)` | `[]` | no | | [permissions\_boundary](#input\_permissions\_boundary) | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | | [subnets](#input\_subnets) | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | [tags](#input\_tags) | A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | diff --git a/irsa.tf b/irsa.tf index 9c5d653a25..9ef8d003c8 100644 --- a/irsa.tf +++ b/irsa.tf @@ -9,7 +9,7 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { count = var.enable_irsa && var.create_eks ? 1 : 0 - client_id_list = [local.sts_principal] + client_id_list = local.sts_principal thumbprint_list = [var.eks_oidc_root_ca_thumbprint] url = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0] diff --git a/local.tf b/local.tf index 9d2d7fdd86..d31f1c929c 100644 --- a/local.tf +++ b/local.tf @@ -44,7 +44,7 @@ locals { ) ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" - sts_principal = "sts.${data.aws_partition.current.dns_suffix}" + sts_principal = compact(concat(["sts.${data.aws_partition.current.dns_suffix}"], var.openid_connect_audiences)) policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" workers_group_defaults_defaults = { diff --git a/variables.tf b/variables.tf index b7d560e402..aca69a8d49 100644 --- a/variables.tf +++ b/variables.tf @@ -393,3 +393,9 @@ variable "wait_for_cluster_timeout" { type = number default = 300 } + +variable "openid_connect_audiences" { + description = "List of OpenID Connect audience client IDs to add to the IRSA provider." + type = list(string) + default = [] +} \ No newline at end of file From 02c9a52f3b728e86bdb7feb537ee4e849f55efe2 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Tue, 31 Aug 2021 12:27:26 +0200 Subject: [PATCH 357/962] Updated CHANGELOG --- CHANGELOG.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd9c24d874..da831927a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,14 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.5.0] - 2021-08-31 +FEATURES: +- Allow users to add more Audiences to OpenID Connect ([#1451](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1451)) + + -## [v17.4.0] - 2021-08-26 +## [v17.4.0] - 2021-08-27 BUG FIXES: - Discourage usage of iam_policy_attachment in example ([#1529](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1529)) - Allow instance `Name` tag to be overwritten ([#1538](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1538)) @@ -425,7 +431,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.4.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.5.0...HEAD +[v17.5.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.4.0...v17.5.0 [v17.4.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.3.0...v17.4.0 [v17.3.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.2.0...v17.3.0 [v17.2.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.1.0...v17.2.0 From e1f54ba73f07a83aae8f73b04a112e24a738acc6 Mon Sep 17 00:00:00 2001 From: Kevin Lefevre Date: Tue, 31 Aug 2021 12:51:42 +0200 Subject: [PATCH 358/962] chore: Extract only tflint file in GH Actions (#1453) --- .github/workflows/pre-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index f5d5d776a8..5a3c0ce7e1 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -92,7 +92,7 @@ jobs: run: | pip install pre-commit curl -Lo ./terraform-docs.tar.gz https://github.com/terraform-docs/terraform-docs/releases/download/v0.13.0/terraform-docs-v0.13.0-$(uname)-amd64.tar.gz && tar -xzf terraform-docs.tar.gz terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ - curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/ + curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip tflint && rm tflint.zip && sudo mv tflint /usr/bin/ - name: Execute pre-commit # Run all pre-commit checks on max version supported if: ${{ matrix.version == needs.getBaseVersion.outputs.maxVersion }} From 8e1d5c11b798259c67f308d7f6c8afc5dfe14e9d Mon Sep 17 00:00:00 2001 From: Olesia Ivanenko Date: Tue, 31 Aug 2021 16:11:43 +0300 Subject: [PATCH 359/962] feat: Tags passed into worker_groups_launch_template extend var.tags for the volumes (#1397) --- examples/launch_templates/main.tf | 5 +++++ local.tf | 2 +- workers_launch_template.tf | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index c01256fe58..cc1cbfb690 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -53,6 +53,11 @@ module "eks" { instance_type = "t3.small" asg_desired_capacity = 2 public_ip = true + tags = [{ + key = "ExtraTag" + value = "TagValue" + propagate_at_launch = true + }] }, { name = "worker-group-2" diff --git a/local.tf b/local.tf index d31f1c929c..fbaa04f312 100644 --- a/local.tf +++ b/local.tf @@ -49,7 +49,7 @@ locals { policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" workers_group_defaults_defaults = { name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. - tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. + tags = [] # A list of maps defining extra tags to be applied to the worker group autoscaling group and volumes. ami_id = "" # AMI ID for the eks linux based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. ami_id_windows = "" # AMI ID for the eks windows based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desired_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 6e14b7dcb0..d1d48186c0 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -547,6 +547,11 @@ resource "aws_launch_template" "workers_launch_template" { )}-eks_asg" }, var.tags, + { + for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : + tag["key"] => tag["value"] + if tag["key"] != "Name" && tag["propagate_at_launch"] + } ) } From 979d62d9b807db10ed3a6327c3554ee480175c62 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Tue, 31 Aug 2021 15:12:00 +0200 Subject: [PATCH 360/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da831927a2..929e88bacc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.6.0] - 2021-08-31 +FEATURES: +- Tags passed into worker_groups_launch_template extend var.tags for the volumes ([#1397](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1397)) + + ## [v17.5.0] - 2021-08-31 FEATURES: @@ -431,7 +437,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.5.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.6.0...HEAD +[v17.6.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.5.0...v17.6.0 [v17.5.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.4.0...v17.5.0 [v17.4.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.3.0...v17.4.0 [v17.3.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.2.0...v17.3.0 From 19ce95d7b69c965de72db5eb90cfb396b9dbc767 Mon Sep 17 00:00:00 2001 From: Olesia Ivanenko Date: Thu, 2 Sep 2021 12:39:39 +0300 Subject: [PATCH 361/962] chore: Update client_id_list argument for OIDC provider (#1561) --- irsa.tf | 2 +- local.tf | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/irsa.tf b/irsa.tf index 9ef8d003c8..8c6e34d030 100644 --- a/irsa.tf +++ b/irsa.tf @@ -9,7 +9,7 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { count = var.enable_irsa && var.create_eks ? 1 : 0 - client_id_list = local.sts_principal + client_id_list = local.client_id_list thumbprint_list = [var.eks_oidc_root_ca_thumbprint] url = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0] diff --git a/local.tf b/local.tf index fbaa04f312..6fb5ccd68d 100644 --- a/local.tf +++ b/local.tf @@ -43,8 +43,9 @@ locals { var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${tonumber(var.cluster_version) >= 1.14 ? var.cluster_version : 1.14}-*" ) - ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" - sts_principal = compact(concat(["sts.${data.aws_partition.current.dns_suffix}"], var.openid_connect_audiences)) + ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" + sts_principal = "sts.${data.aws_partition.current.dns_suffix}" + client_id_list = distinct(compact(concat([local.sts_principal], var.openid_connect_audiences))) policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" workers_group_defaults_defaults = { From bcea0708e6a53904eb30e79c80e2468cec2a374c Mon Sep 17 00:00:00 2001 From: Junaid Ali Date: Thu, 2 Sep 2021 11:28:13 +0100 Subject: [PATCH 362/962] feat: Added throughput support for root and EBS disks (#1445) --- README.md | 4 ++-- examples/launch_templates/main.tf | 17 +++++++++++++++++ local.tf | 2 +- versions.tf | 2 +- workers.tf | 10 ++++++++++ 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 621202c1fe..9107ca7436 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.40.0 | +| [aws](#requirement\_aws) | >= 3.44.0 | | [http](#requirement\_http) | >= 2.4.1 | | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | @@ -152,7 +152,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.40.0 | +| [aws](#provider\_aws) | >= 3.44.0 | | [http](#provider\_http) | >= 2.4.1 | | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index cc1cbfb690..ad71e1382c 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -73,5 +73,22 @@ module "eks" { public_ip = true elastic_inference_accelerator = "eia2.medium" }, + { + name = "worker-group-4" + instance_type = "t3.small" + asg_desired_capacity = 1 + public_ip = true + root_volume_size = 150 + root_volume_type = "gp3" + root_volume_throughput = 300 + additional_ebs_volumes = [ + { + block_device_name = "/dev/xvdb" + volume_size = 100 + volume_type = "gp3" + throughput = 150 + }, + ] + }, ] } diff --git a/local.tf b/local.tf index 6fb5ccd68d..7f4799d418 100644 --- a/local.tf +++ b/local.tf @@ -93,7 +93,7 @@ locals { service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. platform = local.default_platform # Platform of workers. Either "linux" or "windows". - additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults + additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, throughput, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults additional_instance_store_volumes = [] # A list of additional instance store (local disk) volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), virtual_name. warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. diff --git a/versions.tf b/versions.tf index db42ebeb2f..e448b56ecd 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.40.0" + aws = ">= 3.44.0" local = ">= 1.4" kubernetes = ">= 1.11.1" http = { diff --git a/workers.tf b/workers.tf index 31a2ffd568..c73046ea17 100644 --- a/workers.tf +++ b/workers.tf @@ -304,6 +304,11 @@ resource "aws_launch_configuration" "workers" { "root_iops", local.workers_group_defaults["root_iops"], ) + throughput = lookup( + var.worker_groups[count.index], + "root_volume_throughput", + local.workers_group_defaults["root_volume_throughput"], + ) delete_on_termination = true } @@ -327,6 +332,11 @@ resource "aws_launch_configuration" "workers" { "iops", local.workers_group_defaults["root_iops"], ) + throughput = lookup( + ebs_block_device.value, + "throughput", + local.workers_group_defaults["root_volume_throughput"], + ) encrypted = lookup( ebs_block_device.value, "encrypted", From c2bd137152945317124c4cded258f12662d267f4 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Thu, 2 Sep 2021 12:28:28 +0200 Subject: [PATCH 363/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 929e88bacc..f18393ca84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.7.0] - 2021-09-02 +FEATURES: +- Added throughput support for root and EBS disks ([#1445](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1445)) + + ## [v17.6.0] - 2021-08-31 FEATURES: @@ -437,7 +443,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.6.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.7.0...HEAD +[v17.7.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.6.0...v17.7.0 [v17.6.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.5.0...v17.6.0 [v17.5.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.4.0...v17.5.0 [v17.4.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.3.0...v17.4.0 From 7f3b695ef143a806e1afb392b0d78f9411e8bb9a Mon Sep 17 00:00:00 2001 From: Yves-Olivier Laroche Date: Fri, 3 Sep 2021 09:37:00 +0100 Subject: [PATCH 364/962] fix: Put KubeletExtraArgs in double quotes for Windows (#1082) --- templates/userdata_windows.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/userdata_windows.tpl b/templates/userdata_windows.tpl index 61be8e8b11..e8856838f1 100644 --- a/templates/userdata_windows.tpl +++ b/templates/userdata_windows.tpl @@ -4,7 +4,7 @@ ${pre_userdata} [string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" [string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' [string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" -& $EKSBootstrapScriptFile -EKSClusterName ${cluster_name} -KubeletExtraArgs '${kubelet_extra_args}' 3>&1 4>&1 5>&1 6>&1 +& $EKSBootstrapScriptFile -EKSClusterName ${cluster_name} -KubeletExtraArgs "${kubelet_extra_args}" 3>&1 4>&1 5>&1 6>&1 $LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } ${additional_userdata} From a9c29719e8a2c6d7ab2cf961b8d6073d10a7247a Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Fri, 3 Sep 2021 10:45:04 +0200 Subject: [PATCH 365/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f18393ca84..1032332efc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.8.0] - 2021-09-03 +BUG FIXES: +- Put KubeletExtraArgs in double quotes for Windows ([#1082](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1082)) + + ## [v17.7.0] - 2021-09-02 FEATURES: @@ -443,7 +449,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.7.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.8.0...HEAD +[v17.8.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.7.0...v17.8.0 [v17.7.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.6.0...v17.7.0 [v17.6.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.5.0...v17.6.0 [v17.5.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.4.0...v17.5.0 From c2490c5148e11ee8ace3811cea1e61d42680a4b4 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Fri, 3 Sep 2021 16:54:59 +0200 Subject: [PATCH 366/962] feat: Ability to tag just EKS cluster (#1569) --- README.md | 1 + cluster.tf | 5 ++++- variables.tf | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9107ca7436..693e017c13 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no | +| [cluster\_tags](#input\_cluster\_tags) | A map of tags to add to just the eks resource. | `map(string)` | `{}` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | | [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | [create\_fargate\_pod\_execution\_role](#input\_create\_fargate\_pod\_execution\_role) | Controls if the EKS Fargate pod execution IAM role should be created. | `bool` | `true` | no | diff --git a/cluster.tf b/cluster.tf index 13d38a09a7..4e5086c853 100644 --- a/cluster.tf +++ b/cluster.tf @@ -12,7 +12,10 @@ resource "aws_eks_cluster" "this" { enabled_cluster_log_types = var.cluster_enabled_log_types role_arn = local.cluster_iam_role_arn version = var.cluster_version - tags = var.tags + tags = merge( + var.tags, + var.cluster_tags, + ) vpc_config { security_group_ids = compact([local.cluster_security_group_id]) diff --git a/variables.tf b/variables.tf index aca69a8d49..064243605f 100644 --- a/variables.tf +++ b/variables.tf @@ -98,6 +98,12 @@ variable "tags" { default = {} } +variable "cluster_tags" { + description = "A map of tags to add to just the eks resource." + type = map(string) + default = {} +} + variable "vpc_id" { description = "VPC where the cluster and workers will be deployed." type = string From b7413b3e37856a7a7697dc67e79c5e9e17521d9d Mon Sep 17 00:00:00 2001 From: Robert Kozak Date: Fri, 3 Sep 2021 08:07:59 -0700 Subject: [PATCH 367/962] feat: Allow override of timeouts in node_groups (#1552) --- README.md | 1 + local.tf | 1 + modules/node_groups/README.md | 2 ++ modules/node_groups/locals.tf | 1 + modules/node_groups/node_groups.tf | 6 ++++++ modules/node_groups/variables.tf | 5 +++++ node_groups.tf | 1 + variables.tf | 9 ++++++++- 8 files changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 693e017c13..a0638a01f1 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [permissions\_boundary](#input\_permissions\_boundary) | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | | [subnets](#input\_subnets) | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | [tags](#input\_tags) | A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | +| [timeouts](#input\_timeouts) | A map of timeouts for create/update/delete operations. | `map(string)` | `{}` | no | | [vpc\_id](#input\_vpc\_id) | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | | [wait\_for\_cluster\_timeout](#input\_wait\_for\_cluster\_timeout) | A timeout (in seconds) to wait for cluster to be available. | `number` | `300` | no | | [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | diff --git a/local.tf b/local.tf index 7f4799d418..6064876f86 100644 --- a/local.tf +++ b/local.tf @@ -96,6 +96,7 @@ locals { additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, throughput, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults additional_instance_store_volumes = [] # A list of additional instance store (local disk) volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), virtual_name. warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. + timeouts = {} # A map of timeouts for create/update/delete operations # Settings for launch templates root_block_device_name = concat(data.aws_ami.eks_worker.*.root_device_name, [""])[0] # Root device name for Linux workers. If not provided, will assume default Linux AMI was used. diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 1ac6612eab..4811e1a709 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -48,6 +48,7 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | subnets | Subnets to contain workers | list(string) | `var.workers_group_defaults[subnets]` | | version | Kubernetes version | string | Provider default behavior | | taints | Kubernetes node taints | list(map) | empty | +| timeouts | A map of timeouts for create/update/delete operations. | `map(string)` | Provider default behavior | | update_default_version | Whether or not to set the new launch template version the Default | bool | `true` | @@ -89,6 +90,7 @@ No modules. | [node\_groups](#input\_node\_groups) | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | | [node\_groups\_defaults](#input\_node\_groups\_defaults) | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | n/a | yes | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | n/a | yes | +| [timeouts](#input\_timeouts) | A map of timeouts for create/update/delete operations. | `map(string)` | n/a | yes | | [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | [worker\_security\_group\_id](#input\_worker\_security\_group\_id) | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | | [workers\_group\_defaults](#input\_workers\_group\_defaults) | Workers group defaults from parent | `any` | n/a | yes | diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index da8d20e582..6d40e03808 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -24,6 +24,7 @@ locals { pre_userdata = var.workers_group_defaults["pre_userdata"] additional_security_group_ids = var.workers_group_defaults["additional_security_group_ids"] taints = [] + timeouts = var.workers_group_defaults["timeouts"] update_default_version = true ebs_optimized = null }, diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 4e3bf596ad..865f3ae634 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -69,6 +69,12 @@ resource "aws_eks_node_group" "workers" { } } + timeouts { + create = lookup(each.value["timeouts"], "create", null) + update = lookup(each.value["timeouts"], "update", null) + delete = lookup(each.value["timeouts"], "delete", null) + } + version = lookup(each.value, "version", null) labels = merge( diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf index 52209e5ef0..d881968b06 100644 --- a/modules/node_groups/variables.tf +++ b/modules/node_groups/variables.tf @@ -36,6 +36,11 @@ variable "tags" { type = map(string) } +variable "timeouts" { + description = "A map of timeouts for create/update/delete operations." + type = map(string) +} + variable "node_groups_defaults" { description = "map of maps of node groups to create. See \"`node_groups` and `node_groups_defaults` keys\" section in README.md for more details" type = any diff --git a/node_groups.tf b/node_groups.tf index ec483b8f02..2a3580828a 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -7,6 +7,7 @@ module "node_groups" { worker_security_group_id = local.worker_security_group_id worker_additional_security_group_ids = var.worker_additional_security_group_ids tags = var.tags + timeouts = var.timeouts node_groups_defaults = var.node_groups_defaults node_groups = var.node_groups ebs_optimized_not_supported = local.ebs_optimized_not_supported diff --git a/variables.tf b/variables.tf index 064243605f..b2019fafc4 100644 --- a/variables.tf +++ b/variables.tf @@ -104,6 +104,12 @@ variable "cluster_tags" { default = {} } +variable "timeouts" { + description = "A map of timeouts for create/update/delete operations." + type = map(string) + default = {} +} + variable "vpc_id" { description = "VPC where the cluster and workers will be deployed." type = string @@ -404,4 +410,5 @@ variable "openid_connect_audiences" { description = "List of OpenID Connect audience client IDs to add to the IRSA provider." type = list(string) default = [] -} \ No newline at end of file +} + From be71ef203bdbedecaf74015b54f344407347b2e6 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Fri, 3 Sep 2021 17:08:20 +0200 Subject: [PATCH 368/962] Updated CHANGELOG --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1032332efc..11e37795a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.9.0] - 2021-09-03 +FEATURES: +- Allow override of timeouts in node_groups ([#1552](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1552)) +- Ability to tag just EKS cluster ([#1569](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1569)) + + ## [v17.8.0] - 2021-09-03 BUG FIXES: @@ -449,7 +456,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.8.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.9.0...HEAD +[v17.9.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.8.0...v17.9.0 [v17.8.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.7.0...v17.8.0 [v17.7.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.6.0...v17.7.0 [v17.6.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.5.0...v17.6.0 From f23f729980e8610d3e641210ecda9eb29224de86 Mon Sep 17 00:00:00 2001 From: marianobilli <14143004+marianobilli@users.noreply.github.com> Date: Fri, 3 Sep 2021 22:56:17 +0200 Subject: [PATCH 369/962] feat: Added support for update_config in EKS managed node groups (#1560) --- examples/managed_node_groups/main.tf | 3 +++ modules/node_groups/README.md | 2 ++ modules/node_groups/node_groups.tf | 9 +++++++++ 3 files changed, 14 insertions(+) diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index 4c8b3fed6c..b619314756 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -95,6 +95,9 @@ module "eks" { effect = "NO_SCHEDULE" } ] + update_config = { + max_unavailable_percentage = 50 # or set `max_unavailable` + } } } diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 4811e1a709..685c6e5533 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -40,6 +40,8 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | launch\_template_version | The version of the LT to use | string | none | | max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` | | min\_capacity | Min number of workers | number | `var.workers_group_defaults[asg_min_size]` | +| update_config.max\_unavailable\_percentage | Max percentage of unavailable nodes during update. (e.g. 25, 50, etc) | number | `null` if `update_config.max_unavailable` is set | +| update_config.max\_unavailable | Max number of unavailable nodes during update | number | `null` if `update_config.max_unavailable_percentage` is set | | name | Name of the node group. If you don't really need this, we recommend you to use `name_prefix` instead. | string | Will use the autogenerate name prefix | | name_prefix | Name prefix of the node group | string | Auto generated | | pre_userdata | userdata to pre-append to the default userdata. Require `create_launch_template` to be `true`| string | "" | diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 865f3ae634..134d38345c 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -69,6 +69,15 @@ resource "aws_eks_node_group" "workers" { } } + dynamic "update_config" { + for_each = try(each.value.update_config.max_unavailable_percentage > 0, each.value.update_config.max_unavailable > 0, false) ? [true] : [] + + content { + max_unavailable_percentage = try(each.value.update_config.max_unavailable_percentage, null) + max_unavailable = try(each.value.update_config.max_unavailable, null) + } + } + timeouts { create = lookup(each.value["timeouts"], "create", null) update = lookup(each.value["timeouts"], "update", null) From ff0360fd60ee743f220d8f84d562381c217bc719 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Fri, 3 Sep 2021 22:56:31 +0200 Subject: [PATCH 370/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11e37795a2..628257d747 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.10.0] - 2021-09-03 +FEATURES: +- Added support for update_config in EKS managed node groups ([#1560](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1560)) + + ## [v17.9.0] - 2021-09-03 FEATURES: @@ -456,7 +462,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.9.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.10.0...HEAD +[v17.10.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.9.0...v17.10.0 [v17.9.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.8.0...v17.9.0 [v17.8.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.7.0...v17.8.0 [v17.7.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.6.0...v17.7.0 From 40e47a40202e581c6a09b107e47325e50d3afab8 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Sat, 4 Sep 2021 10:46:48 +0200 Subject: [PATCH 371/962] fix: Updated required version of AWS provider to 3.56.0 (#1571) --- README.md | 4 ++-- examples/managed_node_groups/versions.tf | 2 +- modules/node_groups/README.md | 4 ++-- modules/node_groups/versions.tf | 2 +- versions.tf | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a0638a01f1..082b6e463a 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.44.0 | +| [aws](#requirement\_aws) | >= 3.56.0 | | [http](#requirement\_http) | >= 2.4.1 | | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | @@ -152,7 +152,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.44.0 | +| [aws](#provider\_aws) | >= 3.56.0 | | [http](#provider\_http) | >= 2.4.1 | | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | diff --git a/examples/managed_node_groups/versions.tf b/examples/managed_node_groups/versions.tf index 6e29ae8f1b..47f7c05e43 100644 --- a/examples/managed_node_groups/versions.tf +++ b/examples/managed_node_groups/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" + aws = ">= 3.56.0" local = ">= 1.4" random = ">= 2.1" kubernetes = "~> 1.11" diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 685c6e5533..10c4cbaf63 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -59,13 +59,13 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.43.0 | +| [aws](#requirement\_aws) | >= 3.56.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.43.0 | +| [aws](#provider\_aws) | >= 3.56.0 | | [cloudinit](#provider\_cloudinit) | n/a | ## Modules diff --git a/modules/node_groups/versions.tf b/modules/node_groups/versions.tf index c68eb70217..ea2a91d7da 100644 --- a/modules/node_groups/versions.tf +++ b/modules/node_groups/versions.tf @@ -2,6 +2,6 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.43.0" + aws = ">= 3.56.0" } } diff --git a/versions.tf b/versions.tf index e448b56ecd..aa74ab7bf1 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.44.0" + aws = ">= 3.56.0" local = ">= 1.4" kubernetes = ">= 1.11.1" http = { From 013afb0cc6ff6178d72ec8639827ee023e61753b Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Sat, 4 Sep 2021 10:47:46 +0200 Subject: [PATCH 372/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 628257d747..6a872175aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.11.0] - 2021-09-04 +BUG FIXES: +- Updated required version of AWS provider to 3.56.0 ([#1571](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1571)) + + ## [v17.10.0] - 2021-09-03 FEATURES: @@ -462,7 +468,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.10.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.11.0...HEAD +[v17.11.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.10.0...v17.11.0 [v17.10.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.9.0...v17.10.0 [v17.9.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.8.0...v17.9.0 [v17.8.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.7.0...v17.8.0 From 7512f171c9a84a28fc1ddc658f08748ab221145b Mon Sep 17 00:00:00 2001 From: nikitacr7 Date: Mon, 6 Sep 2021 12:25:31 +0300 Subject: [PATCH 373/962] feat: Add ability to tag network-interface using Launch Template (#1563) --- .../launchtemplate.tf | 9 +++++++++ modules/node_groups/launch_template.tf | 14 +++++++++++++ workers_launch_template.tf | 20 +++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf index e66bad1a9f..2494a7a688 100644 --- a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf +++ b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf @@ -80,6 +80,15 @@ resource "aws_launch_template" "default" { } } + # Supplying custom tags to EKS instances ENI's is another use-case for LaunchTemplates + tag_specifications { + resource_type = "network-interface" + + tags = { + CustomTag = "EKS example" + } + } + # Tag the LT itself tags = { CustomTag = "EKS example" diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index f70d2f5838..40bcf17713 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -103,6 +103,20 @@ resource "aws_launch_template" "workers" { ) } + # Supplying custom tags to EKS instances ENI's is another use-case for LaunchTemplates + tag_specifications { + resource_type = "network-interface" + + tags = merge( + var.tags, + { + Name = local.node_groups_names[each.key] + }, + lookup(var.node_groups_defaults, "additional_tags", {}), + lookup(var.node_groups[each.key], "additional_tags", {}) + ) + } + # Tag the LT itself tags = merge( var.tags, diff --git a/workers_launch_template.tf b/workers_launch_template.tf index d1d48186c0..9da5a16d04 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -573,6 +573,26 @@ resource "aws_launch_template" "workers_launch_template" { ) } + tag_specifications { + resource_type = "network-interface" + + tags = merge( + { + "Name" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( + var.worker_groups_launch_template[count.index], + "name", + count.index, + )}-eks_asg" + }, + var.tags, + { + for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : + tag["key"] => tag["value"] + if tag["key"] != "Name" && tag["propagate_at_launch"] + } + ) + } + tags = var.tags lifecycle { From 577e16d167a20b2f7d29e7a36849bda2e964da31 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Mon, 6 Sep 2021 11:25:52 +0200 Subject: [PATCH 374/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a872175aa..39db1a8b87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.12.0] - 2021-09-06 +FEATURES: +- Add ability to tag network-interface using Launch Template ([#1563](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1563)) + + ## [v17.11.0] - 2021-09-04 BUG FIXES: @@ -468,7 +474,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.11.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.12.0...HEAD +[v17.12.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.11.0...v17.12.0 [v17.11.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.10.0...v17.11.0 [v17.10.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.9.0...v17.10.0 [v17.9.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.8.0...v17.9.0 From 752c1836786f4c4ca657b0864d691ec436ed4a6d Mon Sep 17 00:00:00 2001 From: Sungho Spark <73847248+sunghospark-calm@users.noreply.github.com> Date: Mon, 6 Sep 2021 04:40:50 -0700 Subject: [PATCH 375/962] fix: Worker security group handling when worker_create_security_group=false (#1461) --- cluster.tf | 2 +- modules/node_groups/launch_template.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster.tf b/cluster.tf index 4e5086c853..e35d8e864c 100644 --- a/cluster.tf +++ b/cluster.tf @@ -80,7 +80,7 @@ resource "aws_security_group_rule" "cluster_egress_internet" { } resource "aws_security_group_rule" "cluster_https_worker_ingress" { - count = var.cluster_create_security_group && var.create_eks ? 1 : 0 + count = var.cluster_create_security_group && var.create_eks && var.worker_create_security_group ? 1 : 0 description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" security_group_id = local.cluster_security_group_id diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 40bcf17713..84fa7552cf 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -52,7 +52,7 @@ resource "aws_launch_template" "workers" { network_interfaces { associate_public_ip_address = lookup(each.value, "public_ip", null) delete_on_termination = lookup(each.value, "eni_delete", null) - security_groups = flatten([ + security_groups = compact(flatten([ var.worker_security_group_id, var.worker_additional_security_group_ids, lookup( @@ -60,7 +60,7 @@ resource "aws_launch_template" "workers" { "additional_security_group_ids", null, ), - ]) + ])) } # if you want to use a custom AMI From d1d135b13f3075afca457bf4d25986a5d4a47dd7 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Mon, 6 Sep 2021 13:41:09 +0200 Subject: [PATCH 376/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39db1a8b87..ebd0104e58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.13.0] - 2021-09-06 +BUG FIXES: +- Worker security group handling when worker_create_security_group=false ([#1461](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1461)) + + ## [v17.12.0] - 2021-09-06 FEATURES: @@ -474,7 +480,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.12.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.13.0...HEAD +[v17.13.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.12.0...v17.13.0 [v17.12.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.11.0...v17.12.0 [v17.11.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.10.0...v17.11.0 [v17.10.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.9.0...v17.10.0 From 710903170f9d39129bb30c5018eba70ac88a92e5 Mon Sep 17 00:00:00 2001 From: Olesia Ivanenko Date: Mon, 6 Sep 2021 15:04:56 +0300 Subject: [PATCH 377/962] feat: Create SG rule for each new cluster_endpoint_private_access_cidr block (#1549) --- cluster.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster.tf b/cluster.tf index e35d8e864c..7d34556886 100644 --- a/cluster.tf +++ b/cluster.tf @@ -91,13 +91,13 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { } resource "aws_security_group_rule" "cluster_private_access_cidrs_source" { - count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_cidrs != null ? 1 : 0 + for_each = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_cidrs != null ? toset(var.cluster_endpoint_private_access_cidrs) : [] description = "Allow private K8S API ingress from custom CIDR source." type = "ingress" from_port = 443 to_port = 443 protocol = "tcp" - cidr_blocks = var.cluster_endpoint_private_access_cidrs + cidr_blocks = [each.value] security_group_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id } From 7f8232c570786dcf33eff71bc5b5b187e6066db2 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Mon, 6 Sep 2021 14:05:53 +0200 Subject: [PATCH 378/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebd0104e58..12256b7a62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.14.0] - 2021-09-06 +FEATURES: +- Create SG rule for each new cluster_endpoint_private_access_cidr block ([#1549](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1549)) + + ## [v17.13.0] - 2021-09-06 BUG FIXES: @@ -480,7 +486,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.13.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.14.0...HEAD +[v17.14.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.13.0...v17.14.0 [v17.13.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.12.0...v17.13.0 [v17.12.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.11.0...v17.12.0 [v17.11.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.10.0...v17.11.0 From 4a7678d37246ca07630c2c19d9fc4dd873cee2b5 Mon Sep 17 00:00:00 2001 From: DayneD89 Date: Mon, 6 Sep 2021 14:15:38 +0100 Subject: [PATCH 379/962] feat: Added ability to pass different subnets for fargate and the cluster (#1527) --- README.md | 1 + examples/fargate/main.tf | 3 ++- fargate.tf | 2 +- variables.tf | 6 ++++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 082b6e463a..06b05eb3ef 100644 --- a/README.md +++ b/README.md @@ -247,6 +247,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [enable\_irsa](#input\_enable\_irsa) | Whether to create OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no | | [fargate\_pod\_execution\_role\_name](#input\_fargate\_pod\_execution\_role\_name) | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | | [fargate\_profiles](#input\_fargate\_profiles) | Fargate profiles to create. See `fargate_profile` keys section in fargate submodule's README.md for more details | `any` | `{}` | no | +| [fargate\_subnets](#input\_fargate\_subnets) | A list of subnets to place fargate workers within (if different from subnets). | `list(string)` | `[]` | no | | [iam\_path](#input\_iam\_path) | If provided, all IAM roles will be created on this path. | `string` | `"/"` | no | | [kubeconfig\_aws\_authenticator\_additional\_args](#input\_kubeconfig\_aws\_authenticator\_additional\_args) | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | `list(string)` | `[]` | no | | [kubeconfig\_aws\_authenticator\_command](#input\_kubeconfig\_aws\_authenticator\_command) | Command to use to fetch AWS EKS credentials. | `string` | `"aws-iam-authenticator"` | no | diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf index 0697a58c6b..254793fb6e 100644 --- a/examples/fargate/main.tf +++ b/examples/fargate/main.tf @@ -61,7 +61,8 @@ module "eks" { source = "../.." cluster_name = local.cluster_name cluster_version = "1.20" - subnets = module.vpc.private_subnets + subnets = [module.vpc.private_subnets[0], module.vpc.public_subnets[1]] + fargate_subnets = [module.vpc.private_subnets[2]] tags = { Environment = "test" diff --git a/fargate.tf b/fargate.tf index 413c582a9c..f48aa34f01 100644 --- a/fargate.tf +++ b/fargate.tf @@ -8,7 +8,7 @@ module "fargate" { permissions_boundary = var.permissions_boundary iam_path = var.iam_path iam_policy_arn_prefix = local.policy_arn_prefix - subnets = var.subnets + subnets = coalescelist(var.fargate_subnets, var.subnets) tags = var.tags # Hack to ensure ordering of resource creation. diff --git a/variables.tf b/variables.tf index b2019fafc4..9b94c992f2 100644 --- a/variables.tf +++ b/variables.tf @@ -87,6 +87,12 @@ variable "map_users" { default = [] } +variable "fargate_subnets" { + description = "A list of subnets to place fargate workers within (if different from subnets)." + type = list(string) + default = [] +} + variable "subnets" { description = "A list of subnets to place the EKS cluster and workers within." type = list(string) From a9b84558b1abb4ca2a4dd07f571439ccfff79f9d Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Mon, 6 Sep 2021 15:16:04 +0200 Subject: [PATCH 380/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12256b7a62..f86847ccba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.15.0] - 2021-09-06 +FEATURES: +- Added ability to pass different subnets for fargate and the cluster ([#1527](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1527)) + + ## [v17.14.0] - 2021-09-06 FEATURES: @@ -486,7 +492,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.14.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.15.0...HEAD +[v17.15.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.14.0...v17.15.0 [v17.14.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.13.0...v17.14.0 [v17.13.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.12.0...v17.13.0 [v17.12.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.11.0...v17.12.0 From a4aef989cbc669c329ed2682f79dc121eb95d8cd Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Wed, 8 Sep 2021 09:30:10 +0200 Subject: [PATCH 381/962] fix: Fixed coalescelist() with subnets in fargate module (#1576) --- fargate.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fargate.tf b/fargate.tf index f48aa34f01..ddba3bd04b 100644 --- a/fargate.tf +++ b/fargate.tf @@ -8,7 +8,7 @@ module "fargate" { permissions_boundary = var.permissions_boundary iam_path = var.iam_path iam_policy_arn_prefix = local.policy_arn_prefix - subnets = coalescelist(var.fargate_subnets, var.subnets) + subnets = coalescelist(var.fargate_subnets, var.subnets, [""]) tags = var.tags # Hack to ensure ordering of resource creation. From 281e46a0e3d027208559aad4c31ecd05ea560d22 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Wed, 8 Sep 2021 09:30:30 +0200 Subject: [PATCH 382/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f86847ccba..d38679c5b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.16.0] - 2021-09-08 +BUG FIXES: +- Fixed coalescelist() with subnets in fargate module ([#1576](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1576)) + + ## [v17.15.0] - 2021-09-06 FEATURES: @@ -492,7 +498,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.15.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.16.0...HEAD +[v17.16.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.15.0...v17.16.0 [v17.15.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.14.0...v17.15.0 [v17.14.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.13.0...v17.14.0 [v17.13.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.12.0...v17.13.0 From 1d322ebe3f3d6b40fcc1540af4ec7950f7514d47 Mon Sep 17 00:00:00 2001 From: Andrii Barabash Date: Wed, 8 Sep 2021 19:51:20 +0300 Subject: [PATCH 383/962] feat: Added custom AMI support for managed node groups (#1473) --- modules/node_groups/README.md | 1 + modules/node_groups/launch_template.tf | 8 +++++--- modules/node_groups/templates/userdata.sh.tpl | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 10c4cbaf63..81be4df053 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -20,6 +20,7 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | additional\_tags | Additional tags to apply to node group | map(string) | Only `var.tags` applied | | ami\_release\_version | AMI version of workers | string | Provider default behavior | | ami\_type | AMI Type. See Terraform or AWS docs | string | Provider default behavior | +| ami\_id | ID of custom AMI. If you use a custom AMI, you need to supply bootstrap script via user-data or as AMI built-in. | string | Provider default behavior | | capacity\_type | Type of instance capacity to provision. Options are `ON_DEMAND` and `SPOT` | string | Provider default behavior | | create_launch_template | Create and use a default launch template | bool | `false` | | desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` | diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 84fa7552cf..1db59632df 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -9,8 +9,10 @@ data "cloudinit_config" "workers_userdata" { content_type = "text/x-shellscript" content = templatefile("${path.module}/templates/userdata.sh.tpl", { - pre_userdata = each.value["pre_userdata"] - kubelet_extra_args = each.value["kubelet_extra_args"] + pre_userdata = each.value["pre_userdata"] + kubelet_extra_args = each.value["kubelet_extra_args"] + cluster_name = var.cluster_name + run_bootstrap_script = lookup(each.value, "ami_id", null) != null } ) } @@ -64,7 +66,7 @@ resource "aws_launch_template" "workers" { } # if you want to use a custom AMI - # image_id = var.ami_id + image_id = lookup(each.value, "ami_id", null) # If you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then # you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 diff --git a/modules/node_groups/templates/userdata.sh.tpl b/modules/node_groups/templates/userdata.sh.tpl index 3aecd0aabb..097c1168f8 100644 --- a/modules/node_groups/templates/userdata.sh.tpl +++ b/modules/node_groups/templates/userdata.sh.tpl @@ -4,3 +4,6 @@ ${pre_userdata} sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" ${kubelet_extra_args}"' /etc/eks/bootstrap.sh +%{ if run_bootstrap_script } + /etc/eks/bootstrap.sh ${cluster_name} +%{ endif } From 95237e9e9e24185e33ea52f5729671512c81994d Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Wed, 8 Sep 2021 18:51:41 +0200 Subject: [PATCH 384/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d38679c5b2..e7f71b4ca7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.17.0] - 2021-09-08 +FEATURES: +- Added custom AMI support for managed node groups ([#1473](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1473)) + + ## [v17.16.0] - 2021-09-08 BUG FIXES: @@ -498,7 +504,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.16.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.17.0...HEAD +[v17.17.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.16.0...v17.17.0 [v17.16.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.15.0...v17.16.0 [v17.15.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.14.0...v17.15.0 [v17.14.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.13.0...v17.14.0 From a1d959edefe81e49887f68b8ee3d08fdd8f9df4a Mon Sep 17 00:00:00 2001 From: Maxime Brunet Date: Wed, 8 Sep 2021 10:24:47 -0700 Subject: [PATCH 385/962] feat: Add metadata_options for node_groups (#1485) --- modules/node_groups/README.md | 3 ++ modules/node_groups/launch_template.tf | 6 +++ modules/node_groups/locals.tf | 53 ++++++++++++++------------ 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 81be4df053..d80b126b41 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -53,6 +53,9 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | taints | Kubernetes node taints | list(map) | empty | | timeouts | A map of timeouts for create/update/delete operations. | `map(string)` | Provider default behavior | | update_default_version | Whether or not to set the new launch template version the Default | bool | `true` | +| metadata_http_endpoint | The state of the instance metadata service. Requires `create_launch_template` to be `true` | string | `var.workers_group_defaults[metadata_http_endpoint]` | +| metadata_http_tokens | If session tokens are required. Requires `create_launch_template` to be `true` | string | `var.workers_group_defaults[metadata_http_tokens]` | +| metadata_http_put_response_hop_limit | The desired HTTP PUT response hop limit for instance metadata requests. Requires `create_launch_template` to be `true` | number | `var.workers_group_defaults[metadata_http_put_response_hop_limit]` | ## Requirements diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 1db59632df..2cf7cebe27 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -77,6 +77,12 @@ resource "aws_launch_template" "workers" { key_name = lookup(each.value, "key_name", null) + metadata_options { + http_endpoint = lookup(each.value, "metadata_http_endpoint", null) + http_tokens = lookup(each.value, "metadata_http_tokens", null) + http_put_response_hop_limit = lookup(each.value, "metadata_http_put_response_hop_limit", null) + } + # Supplying custom tags to EKS instances is another use-case for LaunchTemplates tag_specifications { resource_type = "instance" diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 6d40e03808..5951b83fc6 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -2,31 +2,34 @@ locals { # Merge defaults and per-group values to make code cleaner node_groups_expanded = { for k, v in var.node_groups : k => merge( { - desired_capacity = var.workers_group_defaults["asg_desired_capacity"] - iam_role_arn = var.default_iam_role_arn - instance_types = [var.workers_group_defaults["instance_type"]] - key_name = var.workers_group_defaults["key_name"] - launch_template_id = var.workers_group_defaults["launch_template_id"] - launch_template_version = var.workers_group_defaults["launch_template_version"] - set_instance_types_on_lt = false - max_capacity = var.workers_group_defaults["asg_max_size"] - min_capacity = var.workers_group_defaults["asg_min_size"] - subnets = var.workers_group_defaults["subnets"] - create_launch_template = false - kubelet_extra_args = var.workers_group_defaults["kubelet_extra_args"] - disk_size = var.workers_group_defaults["root_volume_size"] - disk_type = var.workers_group_defaults["root_volume_type"] - disk_encrypted = var.workers_group_defaults["root_encrypted"] - disk_kms_key_id = var.workers_group_defaults["root_kms_key_id"] - enable_monitoring = var.workers_group_defaults["enable_monitoring"] - eni_delete = var.workers_group_defaults["eni_delete"] - public_ip = var.workers_group_defaults["public_ip"] - pre_userdata = var.workers_group_defaults["pre_userdata"] - additional_security_group_ids = var.workers_group_defaults["additional_security_group_ids"] - taints = [] - timeouts = var.workers_group_defaults["timeouts"] - update_default_version = true - ebs_optimized = null + desired_capacity = var.workers_group_defaults["asg_desired_capacity"] + iam_role_arn = var.default_iam_role_arn + instance_types = [var.workers_group_defaults["instance_type"]] + key_name = var.workers_group_defaults["key_name"] + launch_template_id = var.workers_group_defaults["launch_template_id"] + launch_template_version = var.workers_group_defaults["launch_template_version"] + set_instance_types_on_lt = false + max_capacity = var.workers_group_defaults["asg_max_size"] + min_capacity = var.workers_group_defaults["asg_min_size"] + subnets = var.workers_group_defaults["subnets"] + create_launch_template = false + kubelet_extra_args = var.workers_group_defaults["kubelet_extra_args"] + disk_size = var.workers_group_defaults["root_volume_size"] + disk_type = var.workers_group_defaults["root_volume_type"] + disk_encrypted = var.workers_group_defaults["root_encrypted"] + disk_kms_key_id = var.workers_group_defaults["root_kms_key_id"] + enable_monitoring = var.workers_group_defaults["enable_monitoring"] + eni_delete = var.workers_group_defaults["eni_delete"] + public_ip = var.workers_group_defaults["public_ip"] + pre_userdata = var.workers_group_defaults["pre_userdata"] + additional_security_group_ids = var.workers_group_defaults["additional_security_group_ids"] + taints = [] + timeouts = var.workers_group_defaults["timeouts"] + update_default_version = true + ebs_optimized = null + metadata_http_endpoint = var.workers_group_defaults["metadata_http_endpoint"] + metadata_http_tokens = var.workers_group_defaults["metadata_http_tokens"] + metadata_http_put_response_hop_limit = var.workers_group_defaults["metadata_http_put_response_hop_limit"] }, var.node_groups_defaults, v, From 8d4fdc710802c1f637ffc6dce3cb8f7a4841345a Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Wed, 8 Sep 2021 19:25:05 +0200 Subject: [PATCH 386/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7f71b4ca7..3ec5dfff34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.18.0] - 2021-09-08 +FEATURES: +- Add metadata_options for node_groups ([#1485](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1485)) + + ## [v17.17.0] - 2021-09-08 FEATURES: @@ -504,7 +510,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.17.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.18.0...HEAD +[v17.18.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.17.0...v17.18.0 [v17.17.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.16.0...v17.17.0 [v17.16.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.15.0...v17.16.0 [v17.15.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.14.0...v17.15.0 From 619b4a0d4898a42b75873d3b4e218d293405322d Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Tue, 14 Sep 2021 12:59:40 +0200 Subject: [PATCH 387/962] chore: Updated stalebot config (#1587) --- .github/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index 894f3a0efb..f97e68985d 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -1,9 +1,9 @@ --- # Number of days of inactivity before an issue becomes stale -daysUntilStale: 90 +daysUntilStale: 30 # Number of days of inactivity before an stale issue is closed -daysUntilClose: 30 +daysUntilClose: 7 # Label to use when marking an issue as stale staleLabel: stale From 2bdf7d7dd6e4705fdfa267bed40e147bd9287a21 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Thu, 16 Sep 2021 11:35:44 +0200 Subject: [PATCH 388/962] refactor: Refactoring to match the rest of terraform-aws-modules (#1583) --- .pre-commit-config.yaml | 32 ++- README.md | 144 +++++----- aws_auth.tf | 7 +- data.tf | 16 +- examples/README.md | 11 - examples/_bootstrap/README.md | 60 +++++ examples/_bootstrap/main.tf | 50 ++++ examples/_bootstrap/outputs.tf | 14 + examples/_bootstrap/variables.tf | 0 .../versions.tf | 3 +- examples/basic/main.tf | 138 ---------- examples/basic/variables.tf | 52 ---- examples/bottlerocket/README.md | 74 +++++- examples/bottlerocket/data.tf | 22 -- examples/bottlerocket/main.tf | 72 +++-- examples/{basic => bottlerocket}/outputs.tf | 12 +- examples/bottlerocket/variables.tf | 17 -- .../versions.tf | 5 +- examples/complete/README.md | 73 ++++++ examples/complete/main.tf | 247 ++++++++++++++++++ examples/complete/outputs.tf | 29 ++ examples/complete/variables.tf | 1 + examples/{basic => complete}/versions.tf | 2 +- examples/create_false/main.tf | 30 --- examples/create_false/variables.tf | 3 - examples/fargate/README.md | 68 +++++ examples/fargate/main.tf | 229 +++++++++++----- examples/fargate/outputs.tf | 5 - examples/fargate/variables.tf | 52 ---- examples/fargate/versions.tf | 2 +- examples/instance_refresh/main.tf | 12 +- examples/instance_refresh/outputs.tf | 5 - examples/instance_refresh/variables.tf | 7 +- .../irsa/cluster-autoscaler-chart-values.yaml | 2 +- examples/irsa/irsa.tf | 5 +- examples/irsa/main.tf | 5 +- examples/irsa/outputs.tf | 3 +- examples/irsa/variables.tf | 3 - examples/launch_templates/main.tf | 5 +- examples/launch_templates/outputs.tf | 6 - examples/launch_templates/variables.tf | 4 - .../disk_encryption_policy.tf | 142 +++++----- .../launchtemplate.tf | 24 +- .../main.tf | 5 +- .../outputs.tf | 6 - .../variables.tf | 13 +- examples/managed_node_groups/main.tf | 5 +- examples/managed_node_groups/outputs.tf | 5 - examples/managed_node_groups/variables.tf | 4 - examples/secrets_encryption/main.tf | 5 +- examples/secrets_encryption/outputs.tf | 5 - examples/secrets_encryption/variables.tf | 4 - examples/secrets_encryption/versions.tf | 2 +- examples/spot_instances/main.tf | 61 ----- examples/spot_instances/variables.tf | 4 - fargate.tf | 29 +- irsa.tf | 5 +- kubectl.tf | 3 +- local.tf => locals.tf | 89 +++---- cluster.tf => main.tf | 68 +++-- modules/fargate/README.md | 15 +- modules/fargate/data.tf | 17 -- modules/fargate/fargate.tf | 33 --- modules/fargate/locals.tf | 10 - modules/fargate/main.tf | 67 +++++ modules/fargate/variables.tf | 34 +-- modules/node_groups/README.md | 20 +- .../node_groups/{node_groups.tf => main.tf} | 2 +- modules/node_groups/variables.tf | 10 +- modules/node_groups/versions.tf | 3 +- node_groups.tf | 19 +- outputs.tf | 12 +- variables.tf | 23 +- versions.tf | 1 + workers.tf | 69 +++-- workers_launch_template.tf | 22 +- 76 files changed, 1338 insertions(+), 1025 deletions(-) delete mode 100644 examples/README.md create mode 100644 examples/_bootstrap/README.md create mode 100644 examples/_bootstrap/main.tf create mode 100644 examples/_bootstrap/outputs.tf create mode 100644 examples/_bootstrap/variables.tf rename examples/{spot_instances => _bootstrap}/versions.tf (71%) delete mode 100644 examples/basic/main.tf delete mode 100644 examples/basic/variables.tf delete mode 100644 examples/bottlerocket/data.tf rename examples/{basic => bottlerocket}/outputs.tf (82%) rename examples/{create_false => bottlerocket}/versions.tf (52%) create mode 100644 examples/complete/README.md create mode 100644 examples/complete/main.tf create mode 100644 examples/complete/outputs.tf create mode 100644 examples/complete/variables.tf rename examples/{basic => complete}/versions.tf (85%) delete mode 100644 examples/create_false/main.tf delete mode 100644 examples/create_false/variables.tf create mode 100644 examples/fargate/README.md rename examples/{spot_instances => launch_templates_with_managed_node_groups}/outputs.tf (88%) delete mode 100644 examples/spot_instances/main.tf delete mode 100644 examples/spot_instances/variables.tf rename local.tf => locals.tf (83%) rename cluster.tf => main.tf (79%) delete mode 100644 modules/fargate/data.tf delete mode 100644 modules/fargate/fargate.tf delete mode 100644 modules/fargate/locals.tf create mode 100644 modules/fargate/main.tf rename modules/node_groups/{node_groups.tf => main.tf} (98%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6396765535..4fd440d9b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,26 @@ repos: -- repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.50.0 - hooks: - - id: terraform_fmt - - id: terraform_docs - - id: terraform_validate - - id: terraform_tflint + - repo: git://github.com/antonbabenko/pre-commit-terraform + rev: v1.50.0 + hooks: + - id: terraform_fmt + - id: terraform_validate + - id: terraform_docs + - id: terraform_tflint + args: + - '--args=--only=terraform_deprecated_interpolation' + - '--args=--only=terraform_deprecated_index' + - '--args=--only=terraform_unused_declarations' + - '--args=--only=terraform_comment_syntax' + - '--args=--only=terraform_documented_outputs' + - '--args=--only=terraform_documented_variables' + - '--args=--only=terraform_typed_variables' + - '--args=--only=terraform_module_pinned_source' +# - '--args=--only=terraform_naming_convention' + - '--args=--only=terraform_required_version' + - '--args=--only=terraform_required_providers' + - '--args=--only=terraform_standard_module_structure' + - '--args=--only=terraform_workspace_remote' + - repo: git://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-merge-conflict diff --git a/README.md b/README.md index 06b05eb3ef..505a872ce1 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,54 @@ -# terraform-aws-eks +# AWS EKS Terraform module [![Lint Status](https://github.com/terraform-aws-modules/terraform-aws-eks/workflows/Lint/badge.svg)](https://github.com/terraform-aws-modules/terraform-aws-eks/actions) [![LICENSE](https://img.shields.io/github/license/terraform-aws-modules/terraform-aws-eks)](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/LICENSE) -A terraform module to create a managed Kubernetes cluster on AWS EKS. Available -through the [Terraform registry](https://registry.terraform.io/modules/terraform-aws-modules/eks/aws). -Inspired by and adapted from [this doc](https://www.terraform.io/docs/providers/aws/guides/eks-getting-started.html) -and its [source code](https://github.com/terraform-providers/terraform-provider-aws/tree/master/examples/eks-getting-started). -Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html). +Terraform module which creates Kubernetes cluster resources on AWS EKS. -## Assumptions +## Features -* You want to create an EKS cluster and an autoscaling group of workers for the cluster. -* You want these resources to exist within security groups that allow communication and coordination. These can be user provided or created within the module. -* You've created a Virtual Private Cloud (VPC) and subnets where you intend to put the EKS resources. The VPC satisfies [EKS requirements](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html). +- Create an EKS cluster +- All node types are supported: + - [Managed Node Groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) + - [Self-managed Nodes](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) + - [Fargate](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) +- Support AWS EKS Optimized or Custom AMI +- Create or manage security groups that allow communication and coordination -## Important note - -The `cluster_version` is the required variable. Kubernetes is evolving a lot, and each major version includes new features, fixes, or changes. -**Always check [Kubernetes Release Notes](https://kubernetes.io/docs/setup/release/notes/) before updating the major version.** +## Important note -You also need to ensure your applications and add ons are updated, or workloads could fail after the upgrade is complete. For action, you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). +Kubernetes is evolving a lot, and each minor version includes new features, fixes, or changes. -An example of harming update was the removal of several commonly used, but deprecated APIs, in Kubernetes 1.16. More information on the API removals, see the [Kubernetes blog post](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). +**Always check [Kubernetes Release Notes](https://kubernetes.io/docs/setup/release/notes/) before updating the major version, and [CHANGELOG.md](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/CHANGELOG.md) for all changes in this EKS module.** -By default, this module manages the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready. We implemented a "retry" logic with a fork of the http provider https://github.com/terraform-aws-modules/terraform-provider-http. This fork adds the support of a self-signed CA certificate. The original PR can be found at https://github.com/hashicorp/terraform-provider-http/pull/29. +You also need to ensure that your applications and add ons are updated, or workloads could fail after the upgrade is complete. For action, you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). -Setting `instance_refresh_enabled` to true will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. Find the complete example here [instance_refresh](examples/instance_refresh). ## Usage example -A full example leveraging other community modules is contained in the [examples/basic directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). - ```hcl -data "aws_eks_cluster" "cluster" { - name = module.my-cluster.cluster_id +data "aws_eks_cluster" "eks" { + name = module.eks.cluster_id } -data "aws_eks_cluster_auth" "cluster" { - name = module.my-cluster.cluster_id +data "aws_eks_cluster_auth" "eks" { + name = module.eks.cluster_id } provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) - token = data.aws_eks_cluster_auth.cluster.token - load_config_file = false - version = "~> 1.9" + host = data.aws_eks_cluster.eks.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.eks.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.eks.token } -module "my-cluster" { +module "eks" { source = "terraform-aws-modules/eks/aws" + + cluster_version = "1.21" cluster_name = "my-cluster" - cluster_version = "1.17" - subnets = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] vpc_id = "vpc-1234556abcdef" + subnets = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] worker_groups = [ { @@ -65,56 +58,50 @@ module "my-cluster" { ] } ``` -## Conditional creation -Sometimes you need to have a way to create EKS resources conditionally but Terraform does not allow to use `count` inside `module` block, so the solution is to specify argument `create_eks`. +There is also a [complete example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete) which shows large set of features available in the module. -Using this feature _and_ having `manage_aws_auth=true` (the default) requires to set up the kubernetes provider in a way that allows the data sources to not exist. -```hcl -data "aws_eks_cluster" "cluster" { - count = var.create_eks ? 1 : 0 - name = module.eks.cluster_id -} +## Submodules -data "aws_eks_cluster_auth" "cluster" { - count = var.create_eks ? 1 : 0 - name = module.eks.cluster_id -} +Root module calls these modules which can also be used separately to create independent resources: -# In case of not creating the cluster, this will be an incompletely configured, unused provider, which poses no problem. -provider "kubernetes" { - host = element(concat(data.aws_eks_cluster.cluster[*].endpoint, [""]), 0) - cluster_ca_certificate = base64decode(element(concat(data.aws_eks_cluster.cluster[*].certificate_authority.0.data, [""]), 0)) - token = element(concat(data.aws_eks_cluster_auth.cluster[*].token, [""]), 0) - load_config_file = false - version = "1.10" -} +- [fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/fargate) - creates Fargate profiles, see [examples/fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargat) for detailed examples. + -# This cluster will not be created -module "eks" { - source = "terraform-aws-modules/eks/aws" - create_eks = false - # ... omitted -} -``` +## Notes + +- By default, this module manages the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready. We implemented a "retry" logic with a [fork of the http provider](https://github.com/terraform-aws-modules/terraform-provider-http). This fork adds the support of a self-signed CA certificate. The original PR can be found [here](https://github.com/hashicorp/terraform-provider-http/pull/29). + +- Setting `instance_refresh_enabled = true` will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. Find the complete example here [instance_refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/instance_refresh). -## Other documentation -* [Autoscaling](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/autoscaling.md): How to enable worker node autoscaling. -* [Enable Docker Bridge Network](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/enable-docker-bridge-network.md): How to enable the docker bridge network when using the EKS-optimized AMI, which disables it by default. -* [Spot instances](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/spot-instances.md): How to use spot instances with this module. -* [IAM Permissions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md): Minimum IAM permissions needed to setup EKS Cluster. -* [FAQ](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md): Frequently Asked Questions +## Documentation -## Doc generation +### Official docs -Code formatting and documentation for variables and outputs is generated using [pre-commit-terraform hooks](https://github.com/antonbabenko/pre-commit-terraform) which uses [terraform-docs](https://github.com/segmentio/terraform-docs). +- [Amazon Elastic Kubernetes Service (Amazon EKS)](https://docs.aws.amazon.com/eks/latest/userguide/). -Follow [these instructions](https://github.com/antonbabenko/pre-commit-terraform#how-to-install) to install pre-commit locally. +### Module docs -And install `terraform-docs` with `go get github.com/segmentio/terraform-docs` or `brew install terraform-docs`. +- [Autoscaling](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/autoscaling.md): How to enable worker node autoscaling. +- [Enable Docker Bridge Network](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/enable-docker-bridge-network.md): How to enable the docker bridge network when using the EKS-optimized AMI, which disables it by default. +- [Spot instances](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/spot-instances.md): How to use spot instances with this module. +- [IAM Permissions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md): Minimum IAM permissions needed to setup EKS Cluster. +- [FAQ](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md): Frequently Asked Questions + + +## Examples + +There are detailed examples available for you to see how certain features of this module can be used in a straightforward way. Make sure to check them and run them before opening an issue. [Here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md) you can find the list of the minimum IAM Permissions required to create EKS cluster. + +- [Complete](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete) - Create EKS Cluster with all available workers types in various combinations with many of supported features. +- [Bottlerocket](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/bottlerocket) - Create EKS cluster using [Bottlerocket AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami-bottlerocket.html). +- [Fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate) - Create EKS cluster with [Fargate profiles](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) and attach Fargate profiles to an existing EKS cluster. + ## Contributing @@ -122,16 +109,10 @@ Report issues/questions/feature requests on in the [issues](https://github.com/t Full contributing [guidelines are covered here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/.github/CONTRIBUTING.md). -## Change log - -- The [changelog](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/CHANGELOG.md) captures all important release notes from v11.0.0 -- For older release notes, refer to [changelog.pre-v11.0.0.md](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/CHANGELOG.pre-v11.0.0.md) ## Authors -Created by [Brandon O'Connor](https://github.com/brandoconnor) - brandon@atscale.run. -Maintained by [Max Williams](https://github.com/max-rocket-internet) and [Thierno IB. BARRY](https://github.com/barryib). -Many thanks to [the contributors listed here](https://github.com/terraform-aws-modules/terraform-aws-eks/graphs/contributors)! +This module has been originally created by [Brandon O'Connor](https://github.com/brandoconnor), and was maintained by [Max Williams](https://github.com/max-rocket-internet), [Thierno IB. BARRY](https://github.com/barryib) and many more [contributors listed here](https://github.com/terraform-aws-modules/terraform-aws-eks/graphs/contributors)! ## License @@ -144,6 +125,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.56.0 | +| [cloudinit](#requirement\_cloudinit) | >= 2.0 | | [http](#requirement\_http) | >= 2.4.1 | | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | @@ -236,13 +218,14 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [cluster\_iam\_role\_name](#input\_cluster\_iam\_role\_name) | IAM role name for the cluster. If manage\_cluster\_iam\_resources is set to false, set this to reuse an existing IAM role. If manage\_cluster\_iam\_resources is set to true, set this to force the created role name. | `string` | `""` | no | | [cluster\_log\_kms\_key\_id](#input\_cluster\_log\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `""` | no | | [cluster\_log\_retention\_in\_days](#input\_cluster\_log\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | -| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | `""` | no | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of tags to add to just the eks resource. | `map(string)` | `{}` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | +| [cluster\_version](#input\_cluster\_version) | Kubernetes version to use for the EKS cluster. | `string` | `null` | no | | [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | [create\_fargate\_pod\_execution\_role](#input\_create\_fargate\_pod\_execution\_role) | Controls if the EKS Fargate pod execution IAM role should be created. | `bool` | `true` | no | +| [default\_platform](#input\_default\_platform) | Default platform name. Valid options are `linux` and `windows`. | `string` | `"linux"` | no | | [eks\_oidc\_root\_ca\_thumbprint](#input\_eks\_oidc\_root\_ca\_thumbprint) | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | | [enable\_irsa](#input\_enable\_irsa) | Whether to create OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no | | [fargate\_pod\_execution\_role\_name](#input\_fargate\_pod\_execution\_role\_name) | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | @@ -266,10 +249,9 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [node\_groups\_defaults](#input\_node\_groups\_defaults) | Map of values to be applied to all node groups. See `node_groups` module's documentation for more details | `any` | `{}` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider. | `list(string)` | `[]` | no | | [permissions\_boundary](#input\_permissions\_boundary) | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | -| [subnets](#input\_subnets) | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | +| [subnets](#input\_subnets) | A list of subnets to place the EKS cluster and workers within. | `list(string)` | `[]` | no | | [tags](#input\_tags) | A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | -| [timeouts](#input\_timeouts) | A map of timeouts for create/update/delete operations. | `map(string)` | `{}` | no | -| [vpc\_id](#input\_vpc\_id) | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | +| [vpc\_id](#input\_vpc\_id) | VPC where the cluster and workers will be deployed. | `string` | `null` | no | | [wait\_for\_cluster\_timeout](#input\_wait\_for\_cluster\_timeout) | A timeout (in seconds) to wait for cluster to be available. | `number` | `300` | no | | [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | [worker\_ami\_name\_filter](#input\_worker\_ami\_name\_filter) | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | diff --git a/aws_auth.tf b/aws_auth.tf index 6eb563203d..6de697607f 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -18,7 +18,7 @@ locals { ] auth_worker_roles = [ - for index in range(0, var.create_eks ? local.worker_group_count : 0) : { + for index in range(0, var.create_eks ? local.worker_group_launch_configuration_count : 0) : { worker_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${element( coalescelist( aws_iam_instance_profile.workers.*.role, @@ -61,8 +61,7 @@ locals { } resource "kubernetes_config_map" "aws_auth" { - count = var.create_eks && var.manage_aws_auth ? 1 : 0 - depends_on = [data.http.wait_for_cluster[0]] + count = var.create_eks && var.manage_aws_auth ? 1 : 0 metadata { name = "aws-auth" @@ -88,4 +87,6 @@ resource "kubernetes_config_map" "aws_auth" { mapUsers = yamlencode(var.map_users) mapAccounts = yamlencode(var.map_accounts) } + + depends_on = [data.http.wait_for_cluster[0]] } diff --git a/data.tf b/data.tf index bc80e74a69..fe3b6e33c7 100644 --- a/data.tf +++ b/data.tf @@ -18,7 +18,7 @@ data "aws_iam_policy_document" "workers_assume_role_policy" { } data "aws_ami" "eks_worker" { - count = local.worker_has_linux_ami ? 1 : 0 + count = contains(local.worker_groups_platforms, "linux") ? 1 : 0 filter { name = "name" @@ -31,7 +31,7 @@ data "aws_ami" "eks_worker" { } data "aws_ami" "eks_worker_windows" { - count = local.worker_has_windows_ami ? 1 : 0 + count = contains(local.worker_groups_platforms, "windows") ? 1 : 0 filter { name = "name" @@ -65,11 +65,13 @@ data "aws_iam_policy_document" "cluster_assume_role_policy" { data "aws_iam_role" "custom_cluster_iam_role" { count = var.manage_cluster_iam_resources ? 0 : 1 - name = var.cluster_iam_role_name + + name = var.cluster_iam_role_name } data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" { - count = var.manage_worker_iam_resources ? 0 : local.worker_group_count + count = var.manage_worker_iam_resources ? 0 : local.worker_group_launch_configuration_count + name = lookup( var.worker_groups[count.index], "iam_instance_profile_name", @@ -79,6 +81,7 @@ data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" { data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instance_profile" { count = var.manage_worker_iam_resources ? 0 : local.worker_group_launch_template_count + name = lookup( var.worker_groups_launch_template[count.index], "iam_instance_profile_name", @@ -87,9 +90,10 @@ data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instanc } data "http" "wait_for_cluster" { - count = var.create_eks && var.manage_aws_auth ? 1 : 0 + count = var.create_eks && var.manage_aws_auth ? 1 : 0 + url = format("%s/healthz", aws_eks_cluster.this[0].endpoint) - ca_certificate = base64decode(coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0]) + ca_certificate = base64decode(local.cluster_auth_base64) timeout = var.wait_for_cluster_timeout depends_on = [ diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index 7ee5bebfb9..0000000000 --- a/examples/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Examples - -These serve a few purposes: - -1. Shows developers how to use the module in a straightforward way as integrated with other terraform community supported modules. -2. Serves as the test infrastructure for CI on the project. -3. Provides a simple way to play with the Kubernetes cluster you create. - -## IAM Permissions - -You can see the minimum IAM Permissions required [here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md). diff --git a/examples/_bootstrap/README.md b/examples/_bootstrap/README.md new file mode 100644 index 0000000000..28e34f9c8f --- /dev/null +++ b/examples/_bootstrap/README.md @@ -0,0 +1,60 @@ +# Various bootstrap resources required for other EKS examples + +Configuration in this directory creates some resources required in other EKS examples (such as VPC). + +The resources created here are free (no NAT gateways here) and they can reside in test AWS account. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11 | +| [random](#requirement\_random) | >= 2.1 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.22.0 | +| [random](#provider\_random) | >= 2.1 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | + +## Resources + +| Name | Type | +|------|------| +| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_name](#output\_cluster\_name) | Name of EKS Cluster used in tags for subnets | +| [region](#output\_region) | AWS region | +| [vpc](#output\_vpc) | Complete output of VPC module | + diff --git a/examples/_bootstrap/main.tf b/examples/_bootstrap/main.tf new file mode 100644 index 0000000000..8492741f6b --- /dev/null +++ b/examples/_bootstrap/main.tf @@ -0,0 +1,50 @@ +provider "aws" { + region = local.region +} + +locals { + region = "eu-west-1" + name = "bootstrap-example" + vpc_cidr = "10.0.0.0/16" + + cluster_name = "test-eks-${random_string.suffix.result}" +} + +data "aws_availability_zones" "available" {} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + + azs = data.aws_availability_zones.available.names + public_subnets = [for k, v in data.aws_availability_zones.available.names : cidrsubnet(local.vpc_cidr, 8, k)] + private_subnets = [for k, v in data.aws_availability_zones.available.names : cidrsubnet(local.vpc_cidr, 8, k + 10)] + + # NAT Gateway is disabled in the examples primarily to save costs and be able to recreate VPC faster. + enable_nat_gateway = false + single_nat_gateway = false + + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } +} diff --git a/examples/_bootstrap/outputs.tf b/examples/_bootstrap/outputs.tf new file mode 100644 index 0000000000..87a2e49621 --- /dev/null +++ b/examples/_bootstrap/outputs.tf @@ -0,0 +1,14 @@ +output "region" { + description = "AWS region" + value = local.region +} + +output "cluster_name" { + description = "Name of EKS Cluster used in tags for subnets" + value = local.cluster_name +} + +output "vpc" { + description = "Complete output of VPC module" + value = module.vpc +} diff --git a/examples/_bootstrap/variables.tf b/examples/_bootstrap/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/spot_instances/versions.tf b/examples/_bootstrap/versions.tf similarity index 71% rename from examples/spot_instances/versions.tf rename to examples/_bootstrap/versions.tf index 6e29ae8f1b..5539f13af9 100644 --- a/examples/spot_instances/versions.tf +++ b/examples/_bootstrap/versions.tf @@ -3,8 +3,7 @@ terraform { required_providers { aws = ">= 3.22.0" - local = ">= 1.4" random = ">= 2.1" - kubernetes = "~> 1.11" + kubernetes = ">= 1.11" } } diff --git a/examples/basic/main.tf b/examples/basic/main.tf deleted file mode 100644 index c6257a25eb..0000000000 --- a/examples/basic/main.tf +++ /dev/null @@ -1,138 +0,0 @@ -provider "aws" { - region = var.region -} - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) - token = data.aws_eks_cluster_auth.cluster.token - load_config_file = false -} - -data "aws_availability_zones" "available" { -} - -locals { - cluster_name = "test-eks-${random_string.suffix.result}" -} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -resource "aws_security_group" "worker_group_mgmt_one" { - name_prefix = "worker_group_mgmt_one" - vpc_id = module.vpc.vpc_id - - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - - cidr_blocks = [ - "10.0.0.0/8", - ] - } -} - -resource "aws_security_group" "worker_group_mgmt_two" { - name_prefix = "worker_group_mgmt_two" - vpc_id = module.vpc.vpc_id - - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - - cidr_blocks = [ - "192.168.0.0/16", - ] - } -} - -resource "aws_security_group" "all_worker_mgmt" { - name_prefix = "all_worker_management" - vpc_id = module.vpc.vpc_id - - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - - cidr_blocks = [ - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - ] - } -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 2.47" - - name = "test-vpc" - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } -} - -module "eks" { - source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.20" - subnets = module.vpc.private_subnets - - tags = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - - vpc_id = module.vpc.vpc_id - - worker_groups = [ - { - name = "worker-group-1" - instance_type = "t3.small" - additional_userdata = "echo foo bar" - asg_desired_capacity = 2 - additional_security_group_ids = [aws_security_group.worker_group_mgmt_one.id] - }, - { - name = "worker-group-2" - instance_type = "t3.medium" - additional_userdata = "echo foo bar" - additional_security_group_ids = [aws_security_group.worker_group_mgmt_two.id] - asg_desired_capacity = 1 - }, - ] - - worker_additional_security_group_ids = [aws_security_group.all_worker_mgmt.id] - map_roles = var.map_roles - map_users = var.map_users - map_accounts = var.map_accounts -} diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf deleted file mode 100644 index 7085aeabd4..0000000000 --- a/examples/basic/variables.tf +++ /dev/null @@ -1,52 +0,0 @@ -variable "region" { - default = "us-west-2" -} - -variable "map_accounts" { - description = "Additional AWS account numbers to add to the aws-auth configmap." - type = list(string) - - default = [ - "777777777777", - "888888888888", - ] -} - -variable "map_roles" { - description = "Additional IAM roles to add to the aws-auth configmap." - type = list(object({ - rolearn = string - username = string - groups = list(string) - })) - - default = [ - { - rolearn = "arn:aws:iam::66666666666:role/role1" - username = "role1" - groups = ["system:masters"] - }, - ] -} - -variable "map_users" { - description = "Additional IAM users to add to the aws-auth configmap." - type = list(object({ - userarn = string - username = string - groups = list(string) - })) - - default = [ - { - userarn = "arn:aws:iam::66666666666:user/user1" - username = "user1" - groups = ["system:masters"] - }, - { - userarn = "arn:aws:iam::66666666666:user/user2" - username = "user2" - groups = ["system:masters"] - }, - ] -} diff --git a/examples/bottlerocket/README.md b/examples/bottlerocket/README.md index cc73eb7634..f51a0b6952 100644 --- a/examples/bottlerocket/README.md +++ b/examples/bottlerocket/README.md @@ -1,7 +1,71 @@ -# AWS Bottlerocket based nodes +# AWS EKS cluster running Bottlerocket AMI -This is a minimalistic example that shows how to use functionality of this module to deploy -nodes based on [AWS Bottlerocket container OS](https://github.com/bottlerocket-os/bottlerocket) +Configuration in this directory creates EKS cluster with nodes running [AWS Bottlerocket OS](https://github.com/bottlerocket-os/bottlerocket) -Example is minimalistic by purpose - it shows what knobs to turn to make Bottlerocket work. -Do not use default VPC for your workloads deployment. \ No newline at end of file +This is a minimalistic example which shows what knobs to turn to make Bottlerocket work. + +See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami-bottlerocket.html) for more details. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [random](#requirement\_random) | >= 2.1 | +| [tls](#requirement\_tls) | >= 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.22.0 | +| [random](#provider\_random) | >= 2.1 | +| [tls](#provider\_tls) | >= 2.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | | + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role_policy_attachment.ssm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_key_pair.nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | +| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [tls_private_key.nodes](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | +| [aws_ami.bottlerocket_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_subnet_ids.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet_ids) | data source | +| [aws_vpc.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | +| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | +| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | +| [node\_groups](#output\_node\_groups) | Outputs from node groups | +| [region](#output\_region) | AWS region. | + diff --git a/examples/bottlerocket/data.tf b/examples/bottlerocket/data.tf deleted file mode 100644 index bf380b382c..0000000000 --- a/examples/bottlerocket/data.tf +++ /dev/null @@ -1,22 +0,0 @@ -data "aws_ami" "bottlerocket_ami" { - most_recent = true - owners = ["amazon"] - filter { - name = "name" - values = ["bottlerocket-aws-k8s-${var.k8s_version}-x86_64-*"] - } -} - -data "aws_region" "current" {} - -data "aws_vpc" "default" { - default = true -} - -data "aws_subnet_ids" "default" { - vpc_id = data.aws_vpc.default.id -} - -data "aws_iam_policy" "ssm" { - arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" -} \ No newline at end of file diff --git a/examples/bottlerocket/main.tf b/examples/bottlerocket/main.tf index b254af0a12..9631ef5bc7 100644 --- a/examples/bottlerocket/main.tf +++ b/examples/bottlerocket/main.tf @@ -1,31 +1,27 @@ -terraform { - required_version = ">= 0.13.0" +provider "aws" { + region = local.region } -resource "tls_private_key" "nodes" { - algorithm = "RSA" -} - -resource "aws_key_pair" "nodes" { - key_name = "bottlerocket-nodes" - public_key = tls_private_key.nodes.public_key_openssh +locals { + region = "eu-west-1" + k8s_version = "1.21" } module "eks" { - source = "../.." - cluster_name = "bottlerocket" - cluster_version = var.k8s_version - subnets = data.aws_subnet_ids.default.ids + source = "../.." - vpc_id = data.aws_vpc.default.id + cluster_name = "bottlerocket-${random_string.suffix.result}" + cluster_version = local.k8s_version + + vpc_id = data.aws_vpc.default.id + subnets = data.aws_subnet_ids.default.ids write_kubeconfig = false manage_aws_auth = false worker_groups_launch_template = [ { - name = "bottlerocket-nodes" - # passing bottlerocket ami id + name = "bottlerocket-nodes" ami_id = data.aws_ami.bottlerocket_ami.id instance_type = "t3a.small" asg_desired_capacity = 2 @@ -42,9 +38,9 @@ module "eks" { # we are using this section to pass additional arguments for # userdata template rendering userdata_template_extra_args = { - enable_admin_container = var.enable_admin_container - enable_control_container = var.enable_control_container - aws_region = data.aws_region.current.name + enable_admin_container = false + enable_control_container = true + aws_region = local.region } # example of k8s/kubelet configuration via additional_userdata additional_userdata = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11 | +| [local](#requirement\_local) | >= 1.4 | +| [random](#requirement\_random) | >= 2.1 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.22.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [disabled\_eks](#module\_disabled\_eks) | ../.. | | +| [disabled\_fargate](#module\_disabled\_fargate) | ../../modules/fargate | | +| [disabled\_node\_groups](#module\_disabled\_node\_groups) | ../../modules/node_groups | | +| [eks](#module\_eks) | ../.. | | + +## Resources + +| Name | Type | +|------|------| +| [aws_security_group.all_worker_mgmt](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.worker_group_mgmt_one](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.worker_group_mgmt_two](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [terraform_remote_state.bootstrap](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | +| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | +| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | +| [node\_groups](#output\_node\_groups) | Outputs from node groups | +| [region](#output\_region) | AWS region. | + diff --git a/examples/complete/main.tf b/examples/complete/main.tf new file mode 100644 index 0000000000..c0e2c8b679 --- /dev/null +++ b/examples/complete/main.tf @@ -0,0 +1,247 @@ +provider "aws" { + region = local.region +} + +module "eks" { + source = "../.." + + cluster_name = local.cluster_name + cluster_version = "1.21" + + vpc_id = local.vpc.vpc_id + subnets = [local.vpc.private_subnets[0], local.vpc.public_subnets[1]] + fargate_subnets = [local.vpc.private_subnets[2]] + + worker_additional_security_group_ids = [aws_security_group.all_worker_mgmt.id] + + # Worker groups (using Launch Configurations) + worker_groups = [ + { + name = "worker-group-1" + instance_type = "t3.small" + additional_userdata = "echo foo bar" + asg_desired_capacity = 2 + additional_security_group_ids = [aws_security_group.worker_group_mgmt_one.id] + }, + { + name = "worker-group-2" + instance_type = "t3.medium" + additional_userdata = "echo foo bar" + additional_security_group_ids = [aws_security_group.worker_group_mgmt_two.id] + asg_desired_capacity = 1 + }, + ] + + # Worker groups (using Launch Templates) + worker_groups_launch_template = [ + { + name = "spot-1" + override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] + spot_instance_pools = 4 + asg_max_size = 5 + asg_desired_capacity = 5 + kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" + public_ip = true + }, + ] + + # Managed Node Groups + node_groups_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + } + + node_groups = { + example = { + desired_capacity = 1 + max_capacity = 10 + min_capacity = 1 + + instance_types = ["t3.large"] + capacity_type = "SPOT" + k8s_labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + additional_tags = { + ExtraTag = "example" + } + taints = [ + { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + ] + update_config = { + max_unavailable_percentage = 50 # or set `max_unavailable` + } + } + } + + # Fargate + fargate_profiles = { + default = { + name = "default" + selectors = [ + { + namespace = "kube-system" + labels = { + k8s-app = "kube-dns" + } + }, + { + namespace = "default" + } + ] + + tags = { + Owner = "test" + } + } + } + + # AWS Auth (kubernetes_config_map) + map_roles = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] + + map_users = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] + + map_accounts = [ + "777777777777", + "888888888888", + ] + + tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +#################### +# Disabled creation +#################### + +module "disabled_eks" { + source = "../.." + + create_eks = false +} + +module "disabled_fargate" { + source = "../../modules/fargate" + + create_fargate_pod_execution_role = false +} + +module "disabled_node_groups" { + source = "../../modules/node_groups" + + create_eks = false +} + +############# +# Kubernetes +############# + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} + +################################################################################ +# Supporting resources +################################################################################ + +resource "aws_security_group" "worker_group_mgmt_one" { + name_prefix = "worker_group_mgmt_one" + vpc_id = local.vpc.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + + cidr_blocks = [ + "10.0.0.0/8", + ] + } +} + +resource "aws_security_group" "worker_group_mgmt_two" { + name_prefix = "worker_group_mgmt_two" + vpc_id = local.vpc.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + + cidr_blocks = [ + "192.168.0.0/16", + ] + } +} + +resource "aws_security_group" "all_worker_mgmt" { + name_prefix = "all_worker_management" + vpc_id = local.vpc.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + + cidr_blocks = [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + ] + } +} + + +################################################################################ +# Supporting resources (managed in "_bootstrap" directory) +################################################################################ + +data "terraform_remote_state" "bootstrap" { + backend = "local" + + config = { + path = "../_bootstrap/terraform.tfstate" + } +} + +locals { + region = data.terraform_remote_state.bootstrap.outputs.region + cluster_name = data.terraform_remote_state.bootstrap.outputs.cluster_name + vpc = data.terraform_remote_state.bootstrap.outputs.vpc +} diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf new file mode 100644 index 0000000000..356410219f --- /dev/null +++ b/examples/complete/outputs.tf @@ -0,0 +1,29 @@ +output "region" { + description = "AWS region." + value = local.region +} + +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = module.eks.cluster_security_group_id +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = module.eks.kubeconfig +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = module.eks.config_map_aws_auth +} + +output "node_groups" { + description = "Outputs from node groups" + value = module.eks.node_groups +} diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/examples/complete/variables.tf @@ -0,0 +1 @@ + diff --git a/examples/basic/versions.tf b/examples/complete/versions.tf similarity index 85% rename from examples/basic/versions.tf rename to examples/complete/versions.tf index 6e29ae8f1b..120d873e2f 100644 --- a/examples/basic/versions.tf +++ b/examples/complete/versions.tf @@ -5,6 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = "~> 1.11" + kubernetes = ">= 1.11" } } diff --git a/examples/create_false/main.tf b/examples/create_false/main.tf deleted file mode 100644 index 0afffcd2b4..0000000000 --- a/examples/create_false/main.tf +++ /dev/null @@ -1,30 +0,0 @@ -provider "aws" { - region = var.region -} - -data "aws_eks_cluster" "cluster" { - count = 0 - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - count = 0 - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = element(concat(data.aws_eks_cluster.cluster[*].endpoint, [""]), 0) - cluster_ca_certificate = base64decode(element(concat(data.aws_eks_cluster.cluster[*].certificate_authority.0.data, [""]), 0)) - token = element(concat(data.aws_eks_cluster_auth.cluster[*].token, [""]), 0) - load_config_file = false -} - -module "eks" { - source = "../.." - create_eks = false - cluster_version = "" - - vpc_id = "" - cluster_name = "" - subnets = [] -} diff --git a/examples/create_false/variables.tf b/examples/create_false/variables.tf deleted file mode 100644 index 81b8dbe73e..0000000000 --- a/examples/create_false/variables.tf +++ /dev/null @@ -1,3 +0,0 @@ -variable "region" { - default = "us-west-2" -} diff --git a/examples/fargate/README.md b/examples/fargate/README.md new file mode 100644 index 0000000000..6abd31b524 --- /dev/null +++ b/examples/fargate/README.md @@ -0,0 +1,68 @@ +# AWS EKS Cluster with Fargate profiles + +Configuration in this directory creates EKS cluster with Fargate profiles in two different ways: +- Using a root module, where EKS Cluster and Fargate profiles should be created at once. This is the default behaviour for most users. +- Using `modules/fargate` submodule where Fargate profiles should be attached to the barebone EKS Cluster. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11 | +| [local](#requirement\_local) | >= 1.4 | +| [random](#requirement\_random) | >= 2.1 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.22.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [barebone\_eks](#module\_barebone\_eks) | ../.. | | +| [eks](#module\_eks) | ../.. | | +| [fargate\_profile\_existing\_cluster](#module\_fargate\_profile\_existing\_cluster) | ../../modules/fargate | | + +## Resources + +| Name | Type | +|------|------| +| [aws_eks_cluster.barebone](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.barebone](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [terraform_remote_state.bootstrap](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | +| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | +| [fargate\_profile\_arns](#output\_fargate\_profile\_arns) | Outputs from node groups | +| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | + diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf index 254793fb6e..0901843445 100644 --- a/examples/fargate/main.tf +++ b/examples/fargate/main.tf @@ -1,80 +1,84 @@ -terraform { - required_version = ">= 0.12.6" -} - provider "aws" { - region = var.region + region = local.region } -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} +module "eks" { + source = "../.." -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} + cluster_name = local.cluster_name + cluster_version = "1.21" -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) - token = data.aws_eks_cluster_auth.cluster.token - load_config_file = false -} + vpc_id = local.vpc.vpc_id + subnets = [local.vpc.private_subnets[0], local.vpc.public_subnets[1]] + fargate_subnets = [local.vpc.private_subnets[2]] -data "aws_availability_zones" "available" { -} + fargate_profiles = { + default = { + name = "default" + selectors = [ + { + namespace = "kube-system" + labels = { + k8s-app = "kube-dns" + } + }, + { + namespace = "default" + labels = { + WorkerType = "fargate" + } + } + ] -locals { - cluster_name = "test-eks-${random_string.suffix.result}" -} + tags = { + Owner = "default" + } + } -resource "random_string" "suffix" { - length = 8 - special = false -} + secondary = { + name = "secondary" + selectors = [ + { + namespace = "default" + labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + } + ] -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 2.47" - - name = "test-vpc" - cidr = "172.16.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"] - public_subnets = ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/elb" = "1" - } + # Using specific subnets instead of the ones configured in EKS (`subnets` and `fargate_subnets`) + subnets = [local.vpc.private_subnets[1]] - private_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" + tags = { + Owner = "secondary" + } + } } -} -module "eks" { - source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.20" - subnets = [module.vpc.private_subnets[0], module.vpc.public_subnets[1]] - fargate_subnets = [module.vpc.private_subnets[2]] + manage_aws_auth = false tags = { Environment = "test" GithubRepo = "terraform-aws-eks" GithubOrg = "terraform-aws-modules" } +} + +############################################## +# Calling submodule with existing EKS cluster +############################################## - vpc_id = module.vpc.vpc_id +module "fargate_profile_existing_cluster" { + source = "../../modules/fargate" + + cluster_name = module.barebone_eks.cluster_id + subnets = [local.vpc.private_subnets[0], local.vpc.private_subnets[1]] fargate_profiles = { - default = { - name = "default" + profile1 = { + name = "profile1" selectors = [ { namespace = "kube-system" @@ -82,27 +86,116 @@ module "eks" { k8s-app = "kube-dns" } }, + { + namespace = "profile" + labels = { + WorkerType = "fargate" + } + } + ] + + tags = { + Owner = "profile1" + } + } + + profile2 = { + name = "profile2" + selectors = [ { namespace = "default" - # Kubernetes labels for selection - # labels = { - # Environment = "test" - # GithubRepo = "terraform-aws-eks" - # GithubOrg = "terraform-aws-modules" - # } + labels = { + Fargate = "profile2" + } } ] - # using specific subnets instead of all the ones configured in eks - # subnets = ["subnet-0ca3e3d1234a56c78"] + # Using specific subnets instead of the ones configured in EKS (`subnets` and `fargate_subnets`) + subnets = [local.vpc.private_subnets[1]] tags = { - Owner = "test" + Owner = "profile2" } } } - map_roles = var.map_roles - map_users = var.map_users - map_accounts = var.map_accounts + tags = { + DoYouLoveFargate = "Yes" + } +} + +############# +# Kubernetes +############# + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} + +############################################################ +# Barebone EKS Cluster where submodules can add extra stuff +############################################################ + +module "barebone_eks" { + source = "../.." + + cluster_name = "barebone-${local.cluster_name}" + cluster_version = "1.21" + + vpc_id = local.vpc.vpc_id + subnets = local.vpc.private_subnets + + tags = { + Environment = "test" + Barebone = "yes_please" + } +} + +############# +# Kubernetes +############# + +data "aws_eks_cluster" "barebone" { + name = module.barebone_eks.cluster_id +} + +data "aws_eks_cluster_auth" "barebone" { + name = module.barebone_eks.cluster_id +} + +provider "kubernetes" { + alias = "barebone" + + host = data.aws_eks_cluster.barebone.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.barebone.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.barebone.token +} + + +################################################################################ +# Supporting resources (managed in "_bootstrap" directory) +################################################################################ + +data "terraform_remote_state" "bootstrap" { + backend = "local" + + config = { + path = "../_bootstrap/terraform.tfstate" + } +} + +locals { + region = data.terraform_remote_state.bootstrap.outputs.region + cluster_name = data.terraform_remote_state.bootstrap.outputs.cluster_name + vpc = data.terraform_remote_state.bootstrap.outputs.vpc } diff --git a/examples/fargate/outputs.tf b/examples/fargate/outputs.tf index 59aa57a2c9..b7f23eeaf2 100644 --- a/examples/fargate/outputs.tf +++ b/examples/fargate/outputs.tf @@ -18,11 +18,6 @@ output "config_map_aws_auth" { value = module.eks.config_map_aws_auth } -output "region" { - description = "AWS region." - value = var.region -} - output "fargate_profile_arns" { description = "Outputs from node groups" value = module.eks.fargate_profile_arns diff --git a/examples/fargate/variables.tf b/examples/fargate/variables.tf index 7085aeabd4..e69de29bb2 100644 --- a/examples/fargate/variables.tf +++ b/examples/fargate/variables.tf @@ -1,52 +0,0 @@ -variable "region" { - default = "us-west-2" -} - -variable "map_accounts" { - description = "Additional AWS account numbers to add to the aws-auth configmap." - type = list(string) - - default = [ - "777777777777", - "888888888888", - ] -} - -variable "map_roles" { - description = "Additional IAM roles to add to the aws-auth configmap." - type = list(object({ - rolearn = string - username = string - groups = list(string) - })) - - default = [ - { - rolearn = "arn:aws:iam::66666666666:role/role1" - username = "role1" - groups = ["system:masters"] - }, - ] -} - -variable "map_users" { - description = "Additional IAM users to add to the aws-auth configmap." - type = list(object({ - userarn = string - username = string - groups = list(string) - })) - - default = [ - { - userarn = "arn:aws:iam::66666666666:user/user1" - username = "user1" - groups = ["system:masters"] - }, - { - userarn = "arn:aws:iam::66666666666:user/user2" - username = "user2" - groups = ["system:masters"] - }, - ] -} diff --git a/examples/fargate/versions.tf b/examples/fargate/versions.tf index 6e29ae8f1b..120d873e2f 100644 --- a/examples/fargate/versions.tf +++ b/examples/fargate/versions.tf @@ -5,6 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = "~> 1.11" + kubernetes = ">= 1.11" } } diff --git a/examples/instance_refresh/main.tf b/examples/instance_refresh/main.tf index 1883ecc70c..54735fcddd 100644 --- a/examples/instance_refresh/main.tf +++ b/examples/instance_refresh/main.tf @@ -1,7 +1,7 @@ # Based on the official aws-node-termination-handler setup guide at https://github.com/aws/aws-node-termination-handler#infrastructure-setup provider "aws" { - region = var.region + region = local.region } data "aws_caller_identity" "current" {} @@ -16,15 +16,14 @@ data "aws_eks_cluster_auth" "cluster" { provider "kubernetes" { host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.cluster.token - load_config_file = false } provider "helm" { kubernetes { host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.cluster.token } } @@ -34,6 +33,7 @@ data "aws_availability_zones" "available" { locals { cluster_name = "test-refresh-${random_string.suffix.result}" + region = "eu-west-1" } resource "random_string" "suffix" { @@ -102,7 +102,7 @@ data "aws_iam_policy_document" "aws_node_termination_handler_events" { "sqs:SendMessage", ] resources = [ - "arn:aws:sqs:${var.region}:${data.aws_caller_identity.current.account_id}:${local.cluster_name}", + "arn:aws:sqs:${local.region}:${data.aws_caller_identity.current.account_id}:${local.cluster_name}", ] } } @@ -184,7 +184,7 @@ resource "helm_release" "aws_node_termination_handler" { set { name = "awsRegion" - value = var.region + value = local.region } set { name = "serviceAccount.name" diff --git a/examples/instance_refresh/outputs.tf b/examples/instance_refresh/outputs.tf index 3b981f1f1b..a3dd033616 100644 --- a/examples/instance_refresh/outputs.tf +++ b/examples/instance_refresh/outputs.tf @@ -18,11 +18,6 @@ output "config_map_aws_auth" { value = module.eks.config_map_aws_auth } -output "region" { - description = "AWS region." - value = var.region -} - output "sqs_queue_asg_notification_arn" { description = "SQS queue ASG notification ARN" value = module.aws_node_termination_handler_sqs.sqs_queue_arn diff --git a/examples/instance_refresh/variables.tf b/examples/instance_refresh/variables.tf index 96fc26df92..60b24336b3 100644 --- a/examples/instance_refresh/variables.tf +++ b/examples/instance_refresh/variables.tf @@ -1,18 +1,17 @@ -variable "region" { - default = "us-west-2" -} - variable "aws_node_termination_handler_chart_version" { description = "Version of the aws-node-termination-handler Helm chart to install." + type = string default = "0.15.0" } variable "namespace" { description = "Namespace for the aws-node-termination-handler." + type = string default = "kube-system" } variable "serviceaccount" { description = "Serviceaccount for the aws-node-termination-handler." + type = string default = "aws-node-termination-handler" } diff --git a/examples/irsa/cluster-autoscaler-chart-values.yaml b/examples/irsa/cluster-autoscaler-chart-values.yaml index ccde4365a5..4e5494de30 100644 --- a/examples/irsa/cluster-autoscaler-chart-values.yaml +++ b/examples/irsa/cluster-autoscaler-chart-values.yaml @@ -1,4 +1,4 @@ -awsRegion: us-west-2 +awsRegion: eu-west-1 rbac: create: true diff --git a/examples/irsa/irsa.tf b/examples/irsa/irsa.tf index c841d4f266..7bb9f7f258 100644 --- a/examples/irsa/irsa.tf +++ b/examples/irsa/irsa.tf @@ -1,6 +1,7 @@ module "iam_assumable_role_admin" { - source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "3.6.0" + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "3.6.0" + create_role = true role_name = "cluster-autoscaler" provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index e6c9fa4018..c9ea505900 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -1,5 +1,5 @@ provider "aws" { - region = var.region + region = "eu-west-1" } data "aws_eks_cluster" "cluster" { @@ -12,9 +12,8 @@ data "aws_eks_cluster_auth" "cluster" { provider "kubernetes" { host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.cluster.token - load_config_file = false } data "aws_availability_zones" "available" {} diff --git a/examples/irsa/outputs.tf b/examples/irsa/outputs.tf index ef2ab9577a..796e8ee3d4 100644 --- a/examples/irsa/outputs.tf +++ b/examples/irsa/outputs.tf @@ -1,3 +1,4 @@ output "aws_account_id" { - value = data.aws_caller_identity.current.account_id + description = "IAM AWS account id" + value = data.aws_caller_identity.current.account_id } diff --git a/examples/irsa/variables.tf b/examples/irsa/variables.tf index 81b8dbe73e..e69de29bb2 100644 --- a/examples/irsa/variables.tf +++ b/examples/irsa/variables.tf @@ -1,3 +0,0 @@ -variable "region" { - default = "us-west-2" -} diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index ad71e1382c..68fc20599b 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -1,5 +1,5 @@ provider "aws" { - region = var.region + region = "eu-west-1" } data "aws_eks_cluster" "cluster" { @@ -12,9 +12,8 @@ data "aws_eks_cluster_auth" "cluster" { provider "kubernetes" { host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.cluster.token - load_config_file = false } data "aws_availability_zones" "available" { diff --git a/examples/launch_templates/outputs.tf b/examples/launch_templates/outputs.tf index a0788aff1d..359db3a481 100644 --- a/examples/launch_templates/outputs.tf +++ b/examples/launch_templates/outputs.tf @@ -17,9 +17,3 @@ output "config_map_aws_auth" { description = "A kubernetes configuration to authenticate to this EKS cluster." value = module.eks.config_map_aws_auth } - -output "region" { - description = "AWS region." - value = var.region -} - diff --git a/examples/launch_templates/variables.tf b/examples/launch_templates/variables.tf index f69e50026b..e69de29bb2 100644 --- a/examples/launch_templates/variables.tf +++ b/examples/launch_templates/variables.tf @@ -1,4 +0,0 @@ -variable "region" { - default = "us-west-2" -} - diff --git a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf index 0f51fb1fde..0362193505 100644 --- a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf +++ b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf @@ -4,74 +4,74 @@ resource "aws_iam_service_linked_role" "autoscaling" { description = "Default Service-Linked Role enables access to AWS Services and Resources used or managed by Auto Scaling" } -data "aws_caller_identity" "current" {} - -# This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes -data "aws_iam_policy_document" "ebs_decryption" { - # Copy of default KMS policy that lets you manage it - statement { - sid = "Enable IAM User Permissions" - effect = "Allow" - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] - } - - actions = [ - "kms:*" - ] - - resources = ["*"] - } - - # Required for EKS - statement { - sid = "Allow service-linked role use of the CMK" - effect = "Allow" - - principals { - type = "AWS" - identifiers = [ - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes - module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs - ] - } - - actions = [ - "kms:Encrypt", - "kms:Decrypt", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:DescribeKey" - ] - - resources = ["*"] - } - - statement { - sid = "Allow attachment of persistent resources" - effect = "Allow" - - principals { - type = "AWS" - identifiers = [ - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes - module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs - ] - } - - actions = [ - "kms:CreateGrant" - ] - - resources = ["*"] - - condition { - test = "Bool" - variable = "kms:GrantIsForAWSResource" - values = ["true"] - } - - } -} +#data "aws_caller_identity" "current" {} +# +## This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes +#data "aws_iam_policy_document" "ebs_decryption" { +# # Copy of default KMS policy that lets you manage it +# statement { +# sid = "Enable IAM User Permissions" +# effect = "Allow" +# +# principals { +# type = "AWS" +# identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] +# } +# +# actions = [ +# "kms:*" +# ] +# +# resources = ["*"] +# } +# +# # Required for EKS +# statement { +# sid = "Allow service-linked role use of the CMK" +# effect = "Allow" +# +# principals { +# type = "AWS" +# identifiers = [ +# "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes +# module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs +# ] +# } +# +# actions = [ +# "kms:Encrypt", +# "kms:Decrypt", +# "kms:ReEncrypt*", +# "kms:GenerateDataKey*", +# "kms:DescribeKey" +# ] +# +# resources = ["*"] +# } +# +# statement { +# sid = "Allow attachment of persistent resources" +# effect = "Allow" +# +# principals { +# type = "AWS" +# identifiers = [ +# "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes +# module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs +# ] +# } +# +# actions = [ +# "kms:CreateGrant" +# ] +# +# resources = ["*"] +# +# condition { +# test = "Bool" +# variable = "kms:GrantIsForAWSResource" +# values = ["true"] +# } +# +# } +#} diff --git a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf index 2494a7a688..a393770f76 100644 --- a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf +++ b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf @@ -1,15 +1,15 @@ -data "template_file" "launch_template_userdata" { - template = file("${path.module}/templates/userdata.sh.tpl") - - vars = { - cluster_name = local.cluster_name - endpoint = module.eks.cluster_endpoint - cluster_auth_base64 = module.eks.cluster_certificate_authority_data - - bootstrap_extra_args = "" - kubelet_extra_args = "" - } -} +#data "template_file" "launch_template_userdata" { +# template = file("${path.module}/templates/userdata.sh.tpl") +# +# vars = { +# cluster_name = local.cluster_name +# endpoint = module.eks.cluster_endpoint +# cluster_auth_base64 = module.eks.cluster_certificate_authority_data +# +# bootstrap_extra_args = "" +# kubelet_extra_args = "" +# } +#} # This is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) # there are several more options one could set but you probably dont need to modify them diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf index 8d99dcfee5..78c2b31217 100644 --- a/examples/launch_templates_with_managed_node_groups/main.tf +++ b/examples/launch_templates_with_managed_node_groups/main.tf @@ -1,5 +1,5 @@ provider "aws" { - region = var.region + region = "eu-west-1" } data "aws_eks_cluster" "cluster" { @@ -12,9 +12,8 @@ data "aws_eks_cluster_auth" "cluster" { provider "kubernetes" { host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.cluster.token - load_config_file = false } data "aws_availability_zones" "available" { diff --git a/examples/spot_instances/outputs.tf b/examples/launch_templates_with_managed_node_groups/outputs.tf similarity index 88% rename from examples/spot_instances/outputs.tf rename to examples/launch_templates_with_managed_node_groups/outputs.tf index a0788aff1d..359db3a481 100644 --- a/examples/spot_instances/outputs.tf +++ b/examples/launch_templates_with_managed_node_groups/outputs.tf @@ -17,9 +17,3 @@ output "config_map_aws_auth" { description = "A kubernetes configuration to authenticate to this EKS cluster." value = module.eks.config_map_aws_auth } - -output "region" { - description = "AWS region." - value = var.region -} - diff --git a/examples/launch_templates_with_managed_node_groups/variables.tf b/examples/launch_templates_with_managed_node_groups/variables.tf index 6dcb269f4c..351ffdb9cf 100644 --- a/examples/launch_templates_with_managed_node_groups/variables.tf +++ b/examples/launch_templates_with_managed_node_groups/variables.tf @@ -1,15 +1,6 @@ -variable "region" { - default = "eu-central-1" -} - variable "instance_type" { + description = "Instance type" # Smallest recommended, where ~1.1Gb of 2Gb memory is available for the Kubernetes pods after ‘warming up’ Docker, Kubelet, and OS - default = "t3.small" type = string -} - -variable "kms_key_arn" { - default = "" - description = "KMS key ARN to use if you want to encrypt EKS node root volumes" - type = string + default = "t3.small" } diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index b619314756..826223916f 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -1,5 +1,5 @@ provider "aws" { - region = var.region + region = "eu-west-1" } data "aws_eks_cluster" "cluster" { @@ -12,9 +12,8 @@ data "aws_eks_cluster_auth" "cluster" { provider "kubernetes" { host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.cluster.token - load_config_file = false } data "aws_availability_zones" "available" { diff --git a/examples/managed_node_groups/outputs.tf b/examples/managed_node_groups/outputs.tf index 7010db294f..10a3a96604 100644 --- a/examples/managed_node_groups/outputs.tf +++ b/examples/managed_node_groups/outputs.tf @@ -18,11 +18,6 @@ output "config_map_aws_auth" { value = module.eks.config_map_aws_auth } -output "region" { - description = "AWS region." - value = var.region -} - output "node_groups" { description = "Outputs from node groups" value = module.eks.node_groups diff --git a/examples/managed_node_groups/variables.tf b/examples/managed_node_groups/variables.tf index 7085aeabd4..57853d8b4d 100644 --- a/examples/managed_node_groups/variables.tf +++ b/examples/managed_node_groups/variables.tf @@ -1,7 +1,3 @@ -variable "region" { - default = "us-west-2" -} - variable "map_accounts" { description = "Additional AWS account numbers to add to the aws-auth configmap." type = list(string) diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index 9aebd4cbe5..76fa2b2392 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -1,5 +1,5 @@ provider "aws" { - region = var.region + region = "eu-west-1" } data "aws_eks_cluster" "cluster" { @@ -12,9 +12,8 @@ data "aws_eks_cluster_auth" "cluster" { provider "kubernetes" { host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.cluster.token - load_config_file = false } data "aws_availability_zones" "available" { diff --git a/examples/secrets_encryption/outputs.tf b/examples/secrets_encryption/outputs.tf index 51ddb024a2..359db3a481 100644 --- a/examples/secrets_encryption/outputs.tf +++ b/examples/secrets_encryption/outputs.tf @@ -17,8 +17,3 @@ output "config_map_aws_auth" { description = "A kubernetes configuration to authenticate to this EKS cluster." value = module.eks.config_map_aws_auth } - -output "region" { - description = "AWS region." - value = var.region -} diff --git a/examples/secrets_encryption/variables.tf b/examples/secrets_encryption/variables.tf index 7085aeabd4..57853d8b4d 100644 --- a/examples/secrets_encryption/variables.tf +++ b/examples/secrets_encryption/variables.tf @@ -1,7 +1,3 @@ -variable "region" { - default = "us-west-2" -} - variable "map_accounts" { description = "Additional AWS account numbers to add to the aws-auth configmap." type = list(string) diff --git a/examples/secrets_encryption/versions.tf b/examples/secrets_encryption/versions.tf index 6e29ae8f1b..120d873e2f 100644 --- a/examples/secrets_encryption/versions.tf +++ b/examples/secrets_encryption/versions.tf @@ -5,6 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = "~> 1.11" + kubernetes = ">= 1.11" } } diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf deleted file mode 100644 index fb2ad23ee9..0000000000 --- a/examples/spot_instances/main.tf +++ /dev/null @@ -1,61 +0,0 @@ -provider "aws" { - region = var.region -} - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) - token = data.aws_eks_cluster_auth.cluster.token - load_config_file = false -} - -data "aws_availability_zones" "available" { -} - -locals { - cluster_name = "test-eks-spot-${random_string.suffix.result}" -} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 2.47" - - name = "test-vpc-spot" - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_dns_hostnames = true -} - -module "eks" { - source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.20" - subnets = module.vpc.public_subnets - vpc_id = module.vpc.vpc_id - - worker_groups_launch_template = [ - { - name = "spot-1" - override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] - spot_instance_pools = 4 - asg_max_size = 5 - asg_desired_capacity = 5 - kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" - public_ip = true - }, - ] -} diff --git a/examples/spot_instances/variables.tf b/examples/spot_instances/variables.tf deleted file mode 100644 index f69e50026b..0000000000 --- a/examples/spot_instances/variables.tf +++ /dev/null @@ -1,4 +0,0 @@ -variable "region" { - default = "us-west-2" -} - diff --git a/fargate.tf b/fargate.tf index ddba3bd04b..5526e2eed0 100644 --- a/fargate.tf +++ b/fargate.tf @@ -1,23 +1,16 @@ module "fargate" { - source = "./modules/fargate" - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + source = "./modules/fargate" + create_eks = var.create_eks create_fargate_pod_execution_role = var.create_fargate_pod_execution_role - fargate_pod_execution_role_name = var.fargate_pod_execution_role_name - fargate_profiles = var.fargate_profiles - permissions_boundary = var.permissions_boundary - iam_path = var.iam_path - iam_policy_arn_prefix = local.policy_arn_prefix - subnets = coalescelist(var.fargate_subnets, var.subnets, [""]) - tags = var.tags - # Hack to ensure ordering of resource creation. - # This is a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 - # Do not create node_groups before other resources are ready and removes race conditions - # Ensure these resources are created before "unlocking" the data source. - # Will be removed in Terraform 0.13 - eks_depends_on = [ - aws_eks_cluster.this, - kubernetes_config_map.aws_auth, - ] + cluster_name = local.cluster_name + fargate_pod_execution_role_name = var.fargate_pod_execution_role_name + permissions_boundary = var.permissions_boundary + iam_path = var.iam_path + subnets = coalescelist(var.fargate_subnets, var.subnets, [""]) + + fargate_profiles = var.fargate_profiles + + tags = var.tags } diff --git a/irsa.tf b/irsa.tf index 8c6e34d030..5fc3dc8df7 100644 --- a/irsa.tf +++ b/irsa.tf @@ -8,10 +8,11 @@ # https://github.com/terraform-providers/terraform-provider-aws/issues/10104 resource "aws_iam_openid_connect_provider" "oidc_provider" { - count = var.enable_irsa && var.create_eks ? 1 : 0 + count = var.enable_irsa && var.create_eks ? 1 : 0 + client_id_list = local.client_id_list thumbprint_list = [var.eks_oidc_root_ca_thumbprint] - url = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0] + url = local.cluster_oidc_issuer_url tags = merge( { diff --git a/kubectl.tf b/kubectl.tf index 21021f92c6..b5d6947855 100644 --- a/kubectl.tf +++ b/kubectl.tf @@ -1,5 +1,6 @@ resource "local_file" "kubeconfig" { - count = var.write_kubeconfig && var.create_eks ? 1 : 0 + count = var.write_kubeconfig && var.create_eks ? 1 : 0 + content = local.kubeconfig filename = substr(var.kubeconfig_output_path, -1, 1) == "/" ? "${var.kubeconfig_output_path}kubeconfig_${var.cluster_name}" : var.kubeconfig_output_path file_permission = var.kubeconfig_file_permission diff --git a/local.tf b/locals.tf similarity index 83% rename from local.tf rename to locals.tf index 6064876f86..60472a6058 100644 --- a/local.tf +++ b/locals.tf @@ -1,53 +1,38 @@ locals { - cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id - cluster_primary_security_group_id = var.cluster_version >= 1.14 ? element(concat(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, [""]), 0) : null - cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name - cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) - worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id + # EKS Cluster + cluster_id = coalescelist(aws_eks_cluster.this[*].id, [""])[0] + cluster_arn = coalescelist(aws_eks_cluster.this[*].arn, [""])[0] + cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + cluster_endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] + cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] + cluster_oidc_issuer_url = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc[0].issuer, [""]))[0] + cluster_primary_security_group_id = coalescelist(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, [""])[0] + + cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id + cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name + cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) + + # Worker groups + worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id - default_platform = "linux" default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] default_ami_id_linux = local.workers_group_defaults.ami_id != "" ? local.workers_group_defaults.ami_id : concat(data.aws_ami.eks_worker.*.id, [""])[0] default_ami_id_windows = local.workers_group_defaults.ami_id_windows != "" ? local.workers_group_defaults.ami_id_windows : concat(data.aws_ami.eks_worker_windows.*.id, [""])[0] - kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name - - worker_group_count = length(var.worker_groups) - worker_group_launch_template_count = length(var.worker_groups_launch_template) - - worker_has_linux_ami = length([for x in concat(var.worker_groups, var.worker_groups_launch_template) : x if lookup( - x, - "platform", - # Fallback on default `platform` if it's not defined in current worker group - lookup( - merge({ platform = local.default_platform }, var.workers_group_defaults), - "platform", - null - ) - ) == "linux"]) > 0 - worker_has_windows_ami = length([for x in concat(var.worker_groups, var.worker_groups_launch_template) : x if lookup( - x, - "platform", - # Fallback on default `platform` if it's not defined in current worker group - lookup( - merge({ platform = local.default_platform }, var.workers_group_defaults), - "platform", - null - ) - ) == "windows"]) > 0 + worker_group_launch_configuration_count = length(var.worker_groups) + worker_group_launch_template_count = length(var.worker_groups_launch_template) - worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" - # Windows nodes are available from k8s 1.14. If cluster version is less than 1.14, fix ami filter to some constant to not fail on 'terraform plan'. - worker_ami_name_filter_windows = (var.worker_ami_name_filter_windows != "" ? - var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${tonumber(var.cluster_version) >= 1.14 ? var.cluster_version : 1.14}-*" - ) + worker_groups_platforms = [for x in concat(var.worker_groups, var.worker_groups_launch_template) : try(x.platform, var.workers_group_defaults["platform"], var.default_platform)] - ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" - sts_principal = "sts.${data.aws_partition.current.dns_suffix}" - client_id_list = distinct(compact(concat([local.sts_principal], var.openid_connect_audiences))) + worker_ami_name_filter = coalesce(var.worker_ami_name_filter, "amazon-eks-node-${coalesce(var.cluster_version, "cluster_version")}-v*") + worker_ami_name_filter_windows = coalesce(var.worker_ami_name_filter_windows, "Windows_Server-2019-English-Core-EKS_Optimized-${coalesce(var.cluster_version, "cluster_version")}-*") + ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" + sts_principal = "sts.${data.aws_partition.current.dns_suffix}" + client_id_list = distinct(compact(concat([local.sts_principal], var.openid_connect_audiences))) policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + workers_group_defaults_defaults = { name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. tags = [] # A list of maps defining extra tags to be applied to the worker group autoscaling group and volumes. @@ -92,7 +77,7 @@ locals { placement_group = null # The name of the placement group into which to launch the instances, if any. service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. - platform = local.default_platform # Platform of workers. Either "linux" or "windows". + platform = var.default_platform # Platform of workers. Either "linux" or "windows". additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, throughput, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults additional_instance_store_volumes = [] # A list of additional instance store (local disk) volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), virtual_name. warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. @@ -176,17 +161,17 @@ locals { ] kubeconfig = var.create_eks ? templatefile("${path.module}/templates/kubeconfig.tpl", { - kubeconfig_name = local.kubeconfig_name - endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] - cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] + kubeconfig_name = coalesce(var.kubeconfig_name, "eks_${var.cluster_name}") + endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 aws_authenticator_command = var.kubeconfig_aws_authenticator_command - aws_authenticator_command_args = length(var.kubeconfig_aws_authenticator_command_args) > 0 ? var.kubeconfig_aws_authenticator_command_args : ["token", "-i", coalescelist(aws_eks_cluster.this[*].name, [""])[0]] + aws_authenticator_command_args = coalescelist(var.kubeconfig_aws_authenticator_command_args, ["token", "-i", local.cluster_name]) aws_authenticator_additional_args = var.kubeconfig_aws_authenticator_additional_args aws_authenticator_env_variables = var.kubeconfig_aws_authenticator_env_variables }) : "" - userdata_rendered = [ - for index in range(var.create_eks ? local.worker_group_count : 0) : templatefile( + launch_configuration_userdata_rendered = [ + for index in range(var.create_eks ? local.worker_group_launch_configuration_count : 0) : templatefile( lookup( var.worker_groups[index], "userdata_template_file", @@ -196,9 +181,9 @@ locals { ), merge({ platform = lookup(var.worker_groups[index], "platform", local.workers_group_defaults["platform"]) - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] - endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] - cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] + cluster_name = local.cluster_name + endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 pre_userdata = lookup( var.worker_groups[index], "pre_userdata", @@ -240,9 +225,9 @@ locals { ), merge({ platform = lookup(var.worker_groups_launch_template[index], "platform", local.workers_group_defaults["platform"]) - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] - endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] - cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] + cluster_name = local.cluster_name + endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 pre_userdata = lookup( var.worker_groups_launch_template[index], "pre_userdata", diff --git a/cluster.tf b/main.tf similarity index 79% rename from cluster.tf rename to main.tf index 7d34556886..c5cc718e49 100644 --- a/cluster.tf +++ b/main.tf @@ -1,21 +1,20 @@ resource "aws_cloudwatch_log_group" "this" { - count = length(var.cluster_enabled_log_types) > 0 && var.create_eks ? 1 : 0 + count = length(var.cluster_enabled_log_types) > 0 && var.create_eks ? 1 : 0 + name = "/aws/eks/${var.cluster_name}/cluster" retention_in_days = var.cluster_log_retention_in_days kms_key_id = var.cluster_log_kms_key_id - tags = var.tags + + tags = var.tags } resource "aws_eks_cluster" "this" { - count = var.create_eks ? 1 : 0 + count = var.create_eks ? 1 : 0 + name = var.cluster_name enabled_cluster_log_types = var.cluster_enabled_log_types role_arn = local.cluster_iam_role_arn version = var.cluster_version - tags = merge( - var.tags, - var.cluster_tags, - ) vpc_config { security_group_ids = compact([local.cluster_security_group_id]) @@ -29,11 +28,6 @@ resource "aws_eks_cluster" "this" { service_ipv4_cidr = var.cluster_service_ipv4_cidr } - timeouts { - create = var.cluster_create_timeout - delete = var.cluster_delete_timeout - } - dynamic "encryption_config" { for_each = toset(var.cluster_encryption_config) @@ -45,6 +39,16 @@ resource "aws_eks_cluster" "this" { } } + tags = merge( + var.tags, + var.cluster_tags, + ) + + timeouts { + create = var.cluster_create_timeout + delete = var.cluster_delete_timeout + } + depends_on = [ aws_security_group_rule.cluster_egress_internet, aws_security_group_rule.cluster_https_worker_ingress, @@ -56,10 +60,12 @@ resource "aws_eks_cluster" "this" { } resource "aws_security_group" "cluster" { - count = var.cluster_create_security_group && var.create_eks ? 1 : 0 + count = var.cluster_create_security_group && var.create_eks ? 1 : 0 + name_prefix = var.cluster_name description = "EKS cluster security group." vpc_id = var.vpc_id + tags = merge( var.tags, { @@ -69,7 +75,8 @@ resource "aws_security_group" "cluster" { } resource "aws_security_group_rule" "cluster_egress_internet" { - count = var.cluster_create_security_group && var.create_eks ? 1 : 0 + count = var.cluster_create_security_group && var.create_eks ? 1 : 0 + description = "Allow cluster egress access to the Internet." protocol = "-1" security_group_id = local.cluster_security_group_id @@ -80,7 +87,8 @@ resource "aws_security_group_rule" "cluster_egress_internet" { } resource "aws_security_group_rule" "cluster_https_worker_ingress" { - count = var.cluster_create_security_group && var.create_eks && var.worker_create_security_group ? 1 : 0 + count = var.cluster_create_security_group && var.create_eks && var.worker_create_security_group ? 1 : 0 + description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" security_group_id = local.cluster_security_group_id @@ -91,7 +99,8 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { } resource "aws_security_group_rule" "cluster_private_access_cidrs_source" { - for_each = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_cidrs != null ? toset(var.cluster_endpoint_private_access_cidrs) : [] + for_each = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_cidrs != null ? toset(var.cluster_endpoint_private_access_cidrs) : [] + description = "Allow private K8S API ingress from custom CIDR source." type = "ingress" from_port = 443 @@ -103,7 +112,8 @@ resource "aws_security_group_rule" "cluster_private_access_cidrs_source" { } resource "aws_security_group_rule" "cluster_private_access_sg_source" { - count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_sg != null ? length(var.cluster_endpoint_private_access_sg) : 0 + count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_sg != null ? length(var.cluster_endpoint_private_access_sg) : 0 + description = "Allow private K8S API ingress from custom Security Groups source." type = "ingress" from_port = 443 @@ -115,30 +125,35 @@ resource "aws_security_group_rule" "cluster_private_access_sg_source" { } resource "aws_iam_role" "cluster" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + name_prefix = var.cluster_iam_role_name != "" ? null : var.cluster_name name = var.cluster_iam_role_name != "" ? var.cluster_iam_role_name : null assume_role_policy = data.aws_iam_policy_document.cluster_assume_role_policy.json permissions_boundary = var.permissions_boundary path = var.iam_path force_detach_policies = true - tags = var.tags + + tags = var.tags } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEKSClusterPolicy" role = local.cluster_iam_role_name } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEKSServicePolicy" role = local.cluster_iam_role_name } resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSVPCResourceControllerPolicy" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEKSVPCResourceController" role = local.cluster_iam_role_name } @@ -163,16 +178,19 @@ data "aws_iam_policy_document" "cluster_elb_sl_role_creation" { } resource "aws_iam_policy" "cluster_elb_sl_role_creation" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + name_prefix = "${var.cluster_name}-elb-sl-role-creation" description = "Permissions for EKS to create AWSServiceRoleForElasticLoadBalancing service-linked role" policy = data.aws_iam_policy_document.cluster_elb_sl_role_creation[0].json path = var.iam_path - tags = var.tags + + tags = var.tags } resource "aws_iam_role_policy_attachment" "cluster_elb_sl_role_creation" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + policy_arn = aws_iam_policy.cluster_elb_sl_role_creation[0].arn role = local.cluster_iam_role_name } diff --git a/modules/fargate/README.md b/modules/fargate/README.md index 858ec968b7..c4dfaaf6fd 100644 --- a/modules/fargate/README.md +++ b/modules/fargate/README.md @@ -1,13 +1,15 @@ -# eks `fargate` submodule +# EKS `fargate` submodule Helper submodule to create and manage resources related to `aws_eks_fargate_profile`. -## Assumptions -* Designed for use by the parent module and not directly by end users - ## `fargate_profile` keys + `fargate_profile` is a map of maps. Key of first level will be used as unique value for `for_each` resources and in the `aws_eks_fargate_profile` name. Inner map can take the below values. +## Example + +See example code in `examples/fargate`. + | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | name | Fargate profile name | `string` | Auto generated in the following format `[cluster_name]-fargate-[fargate_profile_map_key]`| no | @@ -42,19 +44,18 @@ No modules. | [aws_iam_role_policy_attachment.eks_fargate_pod](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_policy_document.eks_fargate_pod_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_role.custom_fargate_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster. | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster. | `string` | `""` | no | | [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | [create\_fargate\_pod\_execution\_role](#input\_create\_fargate\_pod\_execution\_role) | Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created. | `bool` | `true` | no | -| [eks\_depends\_on](#input\_eks\_depends\_on) | List of references to other resources this submodule depends on. | `any` | `null` | no | | [fargate\_pod\_execution\_role\_name](#input\_fargate\_pod\_execution\_role\_name) | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | | [fargate\_profiles](#input\_fargate\_profiles) | Fargate profiles to create. See `fargate_profile` keys section in README.md for more details | `any` | `{}` | no | | [iam\_path](#input\_iam\_path) | IAM roles will be created on this path. | `string` | `"/"` | no | -| [iam\_policy\_arn\_prefix](#input\_iam\_policy\_arn\_prefix) | IAM policy prefix with the correct AWS partition. | `string` | n/a | yes | | [permissions\_boundary](#input\_permissions\_boundary) | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | | [subnets](#input\_subnets) | A list of subnets for the EKS Fargate profiles. | `list(string)` | `[]` | no | | [tags](#input\_tags) | A map of tags to add to all resources. | `map(string)` | `{}` | no | diff --git a/modules/fargate/data.tf b/modules/fargate/data.tf deleted file mode 100644 index ee8e0d8e96..0000000000 --- a/modules/fargate/data.tf +++ /dev/null @@ -1,17 +0,0 @@ -data "aws_iam_policy_document" "eks_fargate_pod_assume_role" { - count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["eks-fargate-pods.amazonaws.com"] - } - } -} - -data "aws_iam_role" "custom_fargate_iam_role" { - count = local.create_eks && !var.create_fargate_pod_execution_role ? 1 : 0 - name = var.fargate_pod_execution_role_name -} diff --git a/modules/fargate/fargate.tf b/modules/fargate/fargate.tf deleted file mode 100644 index caa73faf88..0000000000 --- a/modules/fargate/fargate.tf +++ /dev/null @@ -1,33 +0,0 @@ -resource "aws_iam_role" "eks_fargate_pod" { - count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 - name_prefix = format("%s-fargate", substr(var.cluster_name, 0, 24)) - assume_role_policy = data.aws_iam_policy_document.eks_fargate_pod_assume_role[0].json - permissions_boundary = var.permissions_boundary - tags = var.tags - path = var.iam_path -} - -resource "aws_iam_role_policy_attachment" "eks_fargate_pod" { - count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 - policy_arn = "${var.iam_policy_arn_prefix}/AmazonEKSFargatePodExecutionRolePolicy" - role = aws_iam_role.eks_fargate_pod[0].name -} - -resource "aws_eks_fargate_profile" "this" { - for_each = local.create_eks ? local.fargate_profiles_expanded : {} - cluster_name = var.cluster_name - fargate_profile_name = lookup(each.value, "name", format("%s-fargate-%s", var.cluster_name, replace(each.key, "_", "-"))) - pod_execution_role_arn = local.pod_execution_role_arn - subnet_ids = lookup(each.value, "subnets", var.subnets) - tags = each.value.tags - - dynamic "selector" { - for_each = each.value.selectors - content { - namespace = selector.value["namespace"] - labels = lookup(selector.value, "labels", {}) - } - } - - depends_on = [var.eks_depends_on] -} diff --git a/modules/fargate/locals.tf b/modules/fargate/locals.tf deleted file mode 100644 index 18ba96403b..0000000000 --- a/modules/fargate/locals.tf +++ /dev/null @@ -1,10 +0,0 @@ -locals { - create_eks = var.create_eks && length(var.fargate_profiles) > 0 - pod_execution_role_arn = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.arn, [""]), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.arn, [""]), 0) - pod_execution_role_name = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.name, [""]), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.name, [""]), 0) - - fargate_profiles_expanded = { for k, v in var.fargate_profiles : k => merge( - v, - { tags = merge(var.tags, lookup(v, "tags", {})) }, - ) if var.create_eks } -} diff --git a/modules/fargate/main.tf b/modules/fargate/main.tf new file mode 100644 index 0000000000..74e3c9b2e2 --- /dev/null +++ b/modules/fargate/main.tf @@ -0,0 +1,67 @@ +locals { + create_eks = var.create_eks && length(var.fargate_profiles) > 0 + + pod_execution_role_arn = coalescelist(aws_iam_role.eks_fargate_pod.*.arn, data.aws_iam_role.custom_fargate_iam_role.*.arn, [""])[0] + pod_execution_role_name = coalescelist(aws_iam_role.eks_fargate_pod.*.name, data.aws_iam_role.custom_fargate_iam_role.*.name, [""])[0] + + fargate_profiles = { for k, v in var.fargate_profiles : k => v if var.create_eks } +} + +data "aws_partition" "current" {} + +data "aws_iam_policy_document" "eks_fargate_pod_assume_role" { + count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 + + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["eks-fargate-pods.amazonaws.com"] + } + } +} + +data "aws_iam_role" "custom_fargate_iam_role" { + count = local.create_eks && !var.create_fargate_pod_execution_role ? 1 : 0 + + name = var.fargate_pod_execution_role_name +} + +resource "aws_iam_role" "eks_fargate_pod" { + count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 + + name_prefix = format("%s-fargate", substr(var.cluster_name, 0, 24)) + assume_role_policy = data.aws_iam_policy_document.eks_fargate_pod_assume_role[0].json + permissions_boundary = var.permissions_boundary + tags = var.tags + path = var.iam_path +} + +resource "aws_iam_role_policy_attachment" "eks_fargate_pod" { + count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 + + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy" + role = aws_iam_role.eks_fargate_pod[0].name +} + +resource "aws_eks_fargate_profile" "this" { + for_each = local.fargate_profiles + + cluster_name = var.cluster_name + fargate_profile_name = lookup(each.value, "name", format("%s-fargate-%s", var.cluster_name, replace(each.key, "_", "-"))) + pod_execution_role_arn = local.pod_execution_role_arn + subnet_ids = lookup(each.value, "subnets", var.subnets) + + dynamic "selector" { + for_each = each.value.selectors + + content { + namespace = selector.value["namespace"] + labels = lookup(selector.value, "labels", {}) + } + } + + tags = merge(var.tags, lookup(each.value, "tags", {})) +} diff --git a/modules/fargate/variables.tf b/modules/fargate/variables.tf index acfd69bd96..39e2cc68b3 100644 --- a/modules/fargate/variables.tf +++ b/modules/fargate/variables.tf @@ -1,29 +1,25 @@ -variable "cluster_name" { - description = "Name of the EKS cluster." - type = string -} - variable "create_eks" { description = "Controls if EKS resources should be created (it affects almost all resources)" type = bool default = true } -variable "iam_path" { - description = "IAM roles will be created on this path." - type = string - default = "/" +variable "create_fargate_pod_execution_role" { + description = "Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created." + type = bool + default = true } -variable "iam_policy_arn_prefix" { - description = "IAM policy prefix with the correct AWS partition." +variable "cluster_name" { + description = "Name of the EKS cluster." type = string + default = "" } -variable "create_fargate_pod_execution_role" { - description = "Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created." - type = bool - default = true +variable "iam_path" { + description = "IAM roles will be created on this path." + type = string + default = "/" } variable "fargate_pod_execution_role_name" { @@ -55,11 +51,3 @@ variable "tags" { type = map(string) default = {} } - -# Hack for a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 -# Will be removed in Terraform 0.13 with the support of module's `depends_on` https://github.com/hashicorp/terraform/issues/10462 -variable "eks_depends_on" { - description = "List of references to other resources this submodule depends on." - type = any - default = null -} diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index d80b126b41..f004fd19b4 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -1,11 +1,9 @@ -# eks `node_groups` submodule +# EKS `node_groups` submodule Helper submodule to create and manage resources related to `eks_node_groups`. -## Assumptions -* Designed for use by the parent module and not directly by end users - ## Node Groups' IAM Role + The role ARN specified in `var.default_iam_role_arn` will be used by default. In a simple configuration this will be the worker role created by the parent module. `iam_role_arn` must be specified in either `var.node_groups_defaults` or `var.node_groups` if the default parent IAM role is not being created for whatever reason, for example if `manage_worker_iam_resources` is set to false in the parent. @@ -64,13 +62,14 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.56.0 | +| [cloudinit](#requirement\_cloudinit) | >= 2.0 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.56.0 | -| [cloudinit](#provider\_cloudinit) | n/a | +| [cloudinit](#provider\_cloudinit) | >= 2.0 | ## Modules @@ -88,18 +87,17 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [cluster\_name](#input\_cluster\_name) | Name of parent cluster | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Name of parent cluster | `string` | `""` | no | | [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | -| [default\_iam\_role\_arn](#input\_default\_iam\_role\_arn) | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | n/a | yes | +| [default\_iam\_role\_arn](#input\_default\_iam\_role\_arn) | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | `""` | no | | [ebs\_optimized\_not\_supported](#input\_ebs\_optimized\_not\_supported) | List of instance types that do not support EBS optimization | `list(string)` | `[]` | no | | [ng\_depends\_on](#input\_ng\_depends\_on) | List of references to other resources this submodule depends on | `any` | `null` | no | | [node\_groups](#input\_node\_groups) | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | -| [node\_groups\_defaults](#input\_node\_groups\_defaults) | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | n/a | yes | -| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | n/a | yes | -| [timeouts](#input\_timeouts) | A map of timeouts for create/update/delete operations. | `map(string)` | n/a | yes | +| [node\_groups\_defaults](#input\_node\_groups\_defaults) | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | [worker\_security\_group\_id](#input\_worker\_security\_group\_id) | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | -| [workers\_group\_defaults](#input\_workers\_group\_defaults) | Workers group defaults from parent | `any` | n/a | yes | +| [workers\_group\_defaults](#input\_workers\_group\_defaults) | Workers group defaults from parent | `any` | `{}` | no | ## Outputs diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/main.tf similarity index 98% rename from modules/node_groups/node_groups.tf rename to modules/node_groups/main.tf index 134d38345c..7ceafb53fc 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/main.tf @@ -99,7 +99,7 @@ resource "aws_eks_node_group" "workers" { lifecycle { create_before_destroy = true - ignore_changes = [scaling_config.0.desired_size] + ignore_changes = [scaling_config[0].desired_size] } depends_on = [var.ng_depends_on] diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf index d881968b06..d106bb7a65 100644 --- a/modules/node_groups/variables.tf +++ b/modules/node_groups/variables.tf @@ -7,16 +7,19 @@ variable "create_eks" { variable "cluster_name" { description = "Name of parent cluster" type = string + default = "" } variable "default_iam_role_arn" { description = "ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults`" type = string + default = "" } variable "workers_group_defaults" { description = "Workers group defaults from parent" type = any + default = {} } variable "worker_security_group_id" { @@ -34,16 +37,13 @@ variable "worker_additional_security_group_ids" { variable "tags" { description = "A map of tags to add to all resources" type = map(string) -} - -variable "timeouts" { - description = "A map of timeouts for create/update/delete operations." - type = map(string) + default = {} } variable "node_groups_defaults" { description = "map of maps of node groups to create. See \"`node_groups` and `node_groups_defaults` keys\" section in README.md for more details" type = any + default = {} } variable "node_groups" { diff --git a/modules/node_groups/versions.tf b/modules/node_groups/versions.tf index ea2a91d7da..5324b482ab 100644 --- a/modules/node_groups/versions.tf +++ b/modules/node_groups/versions.tf @@ -2,6 +2,7 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.56.0" + aws = ">= 3.56.0" + cloudinit = ">= 2.0" } } diff --git a/node_groups.tf b/node_groups.tf index 2a3580828a..c40f635208 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -1,16 +1,19 @@ module "node_groups" { - source = "./modules/node_groups" - create_eks = var.create_eks - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + source = "./modules/node_groups" + + create_eks = var.create_eks + + cluster_name = local.cluster_name default_iam_role_arn = coalescelist(aws_iam_role.workers[*].arn, [""])[0] + ebs_optimized_not_supported = local.ebs_optimized_not_supported workers_group_defaults = local.workers_group_defaults worker_security_group_id = local.worker_security_group_id worker_additional_security_group_ids = var.worker_additional_security_group_ids - tags = var.tags - timeouts = var.timeouts - node_groups_defaults = var.node_groups_defaults - node_groups = var.node_groups - ebs_optimized_not_supported = local.ebs_optimized_not_supported + + node_groups_defaults = var.node_groups_defaults + node_groups = var.node_groups + + tags = var.tags # Hack to ensure ordering of resource creation. # This is a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 diff --git a/outputs.tf b/outputs.tf index f6c53513d9..8676285368 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,6 +1,6 @@ output "cluster_id" { description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready." - value = element(concat(aws_eks_cluster.this.*.id, [""]), 0) + value = local.cluster_id # So that calling plans wait for the cluster to be available before attempting to use it. # There is no need to duplicate this datasource @@ -9,17 +9,17 @@ output "cluster_id" { output "cluster_arn" { description = "The Amazon Resource Name (ARN) of the cluster." - value = element(concat(aws_eks_cluster.this.*.arn, [""]), 0) + value = local.cluster_arn } output "cluster_certificate_authority_data" { description = "Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster." - value = element(concat(aws_eks_cluster.this[*].certificate_authority[0].data, [""]), 0) + value = local.cluster_auth_base64 } output "cluster_endpoint" { description = "The endpoint for your EKS Kubernetes API." - value = element(concat(aws_eks_cluster.this.*.endpoint, [""]), 0) + value = local.cluster_endpoint } output "cluster_version" { @@ -49,7 +49,7 @@ output "cluster_iam_role_arn" { output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster OIDC Issuer" - value = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0] + value = local.cluster_oidc_issuer_url } output "cluster_primary_security_group_id" { @@ -109,7 +109,7 @@ output "workers_asg_names" { output "workers_user_data" { description = "User data of worker groups" value = concat( - local.userdata_rendered, + local.launch_configuration_userdata_rendered, local.launch_template_userdata_rendered, ) } diff --git a/variables.tf b/variables.tf index 9b94c992f2..6454e6bad7 100644 --- a/variables.tf +++ b/variables.tf @@ -1,24 +1,25 @@ variable "cluster_enabled_log_types" { - default = [] description = "A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html)" type = list(string) + default = [] } variable "cluster_log_kms_key_id" { - default = "" description = "If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html)" type = string + default = "" } variable "cluster_log_retention_in_days" { - default = 90 description = "Number of days to retain log events. Default retention - 90 days." type = number + default = 90 } variable "cluster_name" { description = "Name of the EKS cluster. Also used as a prefix in names of related resources." type = string + default = "" } variable "cluster_security_group_id" { @@ -30,6 +31,7 @@ variable "cluster_security_group_id" { variable "cluster_version" { description = "Kubernetes version to use for the EKS cluster." type = string + default = null } variable "kubeconfig_output_path" { @@ -50,8 +52,15 @@ variable "write_kubeconfig" { default = true } +variable "default_platform" { + description = "Default platform name. Valid options are `linux` and `windows`." + type = string + default = "linux" +} + variable "manage_aws_auth" { description = "Whether to apply the aws-auth configmap file." + type = bool default = true } @@ -96,6 +105,7 @@ variable "fargate_subnets" { variable "subnets" { description = "A list of subnets to place the EKS cluster and workers within." type = list(string) + default = [] } variable "tags" { @@ -110,15 +120,10 @@ variable "cluster_tags" { default = {} } -variable "timeouts" { - description = "A map of timeouts for create/update/delete operations." - type = map(string) - default = {} -} - variable "vpc_id" { description = "VPC where the cluster and workers will be deployed." type = string + default = null } variable "worker_groups" { diff --git a/versions.tf b/versions.tf index aa74ab7bf1..e9b6bc9562 100644 --- a/versions.tf +++ b/versions.tf @@ -5,6 +5,7 @@ terraform { aws = ">= 3.56.0" local = ">= 1.4" kubernetes = ">= 1.11.1" + cloudinit = ">= 2.0" http = { source = "terraform-aws-modules/http" version = ">= 2.4.1" diff --git a/workers.tf b/workers.tf index c73046ea17..f2cd7cbf1b 100644 --- a/workers.tf +++ b/workers.tf @@ -1,12 +1,13 @@ # Worker Groups using Launch Configurations resource "aws_autoscaling_group" "workers" { - count = var.create_eks ? local.worker_group_count : 0 + count = var.create_eks ? local.worker_group_launch_configuration_count : 0 + name_prefix = join( "-", compact( [ - coalescelist(aws_eks_cluster.this[*].name, [""])[0], + local.cluster_name, lookup(var.worker_groups[count.index], "name", count.index) ] ) @@ -131,16 +132,16 @@ resource "aws_autoscaling_group" "workers" { [ { "key" = "Name" - "value" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg" + "value" = "${local.cluster_name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg" "propagate_at_launch" = true }, { - "key" = "kubernetes.io/cluster/${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}" + "key" = "kubernetes.io/cluster/${local.cluster_name}" "value" = "owned" "propagate_at_launch" = true }, { - "key" = "k8s.io/cluster/${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}" + "key" = "k8s.io/cluster/${local.cluster_name}" "value" = "owned" "propagate_at_launch" = true }, @@ -201,8 +202,9 @@ resource "aws_autoscaling_group" "workers" { } resource "aws_launch_configuration" "workers" { - count = var.create_eks ? local.worker_group_count : 0 - name_prefix = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup(var.worker_groups[count.index], "name", count.index)}" + count = var.create_eks ? local.worker_group_launch_configuration_count : 0 + + name_prefix = "${local.cluster_name}-${lookup(var.worker_groups[count.index], "name", count.index)}" associate_public_ip_address = lookup( var.worker_groups[count.index], "public_ip", @@ -236,7 +238,7 @@ resource "aws_launch_configuration" "workers" { "key_name", local.workers_group_defaults["key_name"], ) - user_data_base64 = base64encode(local.userdata_rendered[count.index]) + user_data_base64 = base64encode(local.launch_configuration_userdata_rendered[count.index]) ebs_optimized = lookup( var.worker_groups[count.index], "ebs_optimized", @@ -368,7 +370,8 @@ resource "aws_launch_configuration" "workers" { } resource "aws_security_group" "workers" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 + name_prefix = var.cluster_name description = "Security group for all nodes in the cluster." vpc_id = var.vpc_id @@ -382,7 +385,8 @@ resource "aws_security_group" "workers" { } resource "aws_security_group_rule" "workers_egress_internet" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 + description = "Allow nodes all egress to the Internet." protocol = "-1" security_group_id = local.worker_security_group_id @@ -393,7 +397,8 @@ resource "aws_security_group_rule" "workers_egress_internet" { } resource "aws_security_group_rule" "workers_ingress_self" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 + description = "Allow node to communicate with each other." protocol = "-1" security_group_id = local.worker_security_group_id @@ -404,7 +409,8 @@ resource "aws_security_group_rule" "workers_ingress_self" { } resource "aws_security_group_rule" "workers_ingress_cluster" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 + description = "Allow workers pods to receive communication from the cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id @@ -415,7 +421,8 @@ resource "aws_security_group_rule" "workers_ingress_cluster" { } resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { - count = var.worker_create_security_group && var.create_eks ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 + count = var.worker_create_security_group && var.create_eks ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 + description = "Allow workers Kubelets to receive communication from the cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id @@ -426,7 +433,8 @@ resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { } resource "aws_security_group_rule" "workers_ingress_cluster_https" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 + description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id @@ -437,7 +445,8 @@ resource "aws_security_group_rule" "workers_ingress_cluster_https" { } resource "aws_security_group_rule" "workers_ingress_cluster_primary" { - count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.create_eks ? 1 : 0 + description = "Allow pods running on workers to receive communication from cluster primary security group (e.g. Fargate pods)." protocol = "all" security_group_id = local.worker_security_group_id @@ -448,7 +457,8 @@ resource "aws_security_group_rule" "workers_ingress_cluster_primary" { } resource "aws_security_group_rule" "cluster_primary_ingress_workers" { - count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_eks ? 1 : 0 + count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.create_eks ? 1 : 0 + description = "Allow pods running on workers to send communication to cluster primary security group (e.g. Fargate pods)." protocol = "all" security_group_id = local.cluster_primary_security_group_id @@ -459,26 +469,29 @@ resource "aws_security_group_rule" "cluster_primary_ingress_workers" { } resource "aws_iam_role" "workers" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - name_prefix = var.workers_role_name != "" ? null : coalescelist(aws_eks_cluster.this[*].name, [""])[0] + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + + name_prefix = var.workers_role_name != "" ? null : local.cluster_name name = var.workers_role_name != "" ? var.workers_role_name : null assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json permissions_boundary = var.permissions_boundary path = var.iam_path force_detach_policies = true - tags = var.tags + + tags = var.tags } resource "aws_iam_instance_profile" "workers" { - count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_count : 0 - name_prefix = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_launch_configuration_count : 0 + + name_prefix = local.cluster_name role = lookup( var.worker_groups[count.index], "iam_role_id", local.default_iam_role_id, ) - path = var.iam_path + tags = var.tags lifecycle { @@ -487,25 +500,29 @@ resource "aws_iam_instance_profile" "workers" { } resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEKSWorkerNodePolicy" role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { - count = var.manage_worker_iam_resources && var.attach_worker_cni_policy && var.create_eks ? 1 : 0 + count = var.manage_worker_iam_resources && var.attach_worker_cni_policy && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEKS_CNI_Policy" role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEC2ContainerRegistryReadOnly" role = aws_iam_role.workers[0].name } resource "aws_iam_role_policy_attachment" "workers_additional_policies" { - count = var.manage_worker_iam_resources && var.create_eks ? length(var.workers_additional_policies) : 0 + count = var.manage_worker_iam_resources && var.create_eks ? length(var.workers_additional_policies) : 0 + role = aws_iam_role.workers[0].name policy_arn = var.workers_additional_policies[count.index] } diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 9da5a16d04..b675304010 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -2,11 +2,12 @@ resource "aws_autoscaling_group" "workers_launch_template" { count = var.create_eks ? local.worker_group_launch_template_count : 0 + name_prefix = join( "-", compact( [ - coalescelist(aws_eks_cluster.this[*].name, [""])[0], + local.cluster_name, lookup(var.worker_groups_launch_template[count.index], "name", count.index) ] ) @@ -219,7 +220,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { [ { "key" = "Name" - "value" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( + "value" = "${local.cluster_name}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -227,7 +228,7 @@ resource "aws_autoscaling_group" "workers_launch_template" { "propagate_at_launch" = true }, { - "key" = "kubernetes.io/cluster/${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}" + "key" = "kubernetes.io/cluster/${local.cluster_name}" "value" = "owned" "propagate_at_launch" = true }, @@ -289,7 +290,8 @@ resource "aws_autoscaling_group" "workers_launch_template" { resource "aws_launch_template" "workers_launch_template" { count = var.create_eks ? (local.worker_group_launch_template_count) : 0 - name_prefix = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( + + name_prefix = "${local.cluster_name}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -540,7 +542,7 @@ resource "aws_launch_template" "workers_launch_template" { tags = merge( { - "Name" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( + "Name" = "${local.cluster_name}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -560,7 +562,7 @@ resource "aws_launch_template" "workers_launch_template" { tags = merge( { - "Name" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( + "Name" = "${local.cluster_name}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -578,7 +580,7 @@ resource "aws_launch_template" "workers_launch_template" { tags = merge( { - "Name" = "${coalescelist(aws_eks_cluster.this[*].name, [""])[0]}-${lookup( + "Name" = "${local.cluster_name}-${lookup( var.worker_groups_launch_template[count.index], "name", count.index, @@ -617,14 +619,16 @@ resource "aws_launch_template" "workers_launch_template" { } resource "aws_iam_instance_profile" "workers_launch_template" { - count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_launch_template_count : 0 - name_prefix = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_launch_template_count : 0 + + name_prefix = local.cluster_name role = lookup( var.worker_groups_launch_template[count.index], "iam_role_id", local.default_iam_role_id, ) path = var.iam_path + tags = var.tags lifecycle { From 926af35595d1e8747d6a41f8de445316400fbbc8 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Thu, 16 Sep 2021 11:36:11 +0200 Subject: [PATCH 389/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ec5dfff34..d0638673a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.19.0] - 2021-09-16 +REFACTORS: +- Refactoring to match the rest of terraform-aws-modules ([#1583](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1583)) + + ## [v17.18.0] - 2021-09-08 FEATURES: @@ -510,7 +516,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.18.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.19.0...HEAD +[v17.19.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.18.0...v17.19.0 [v17.18.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.17.0...v17.18.0 [v17.17.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.16.0...v17.17.0 [v17.16.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.15.0...v17.16.0 From 46f16d6ae842a71f96501b1cfad4416f6f572627 Mon Sep 17 00:00:00 2001 From: Marco Kilchhofer Date: Fri, 17 Sep 2021 16:45:49 +0200 Subject: [PATCH 390/962] feat: Ability to specify cluster update timeout (#1588) --- README.md | 1 + main.tf | 1 + variables.tf | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 505a872ce1..5562a605f1 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of tags to add to just the eks resource. | `map(string)` | `{}` | no | +| [cluster\_update\_timeout](#input\_cluster\_update\_timeout) | Timeout value when updating the EKS cluster. | `string` | `"60m"` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes version to use for the EKS cluster. | `string` | `null` | no | | [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | [create\_fargate\_pod\_execution\_role](#input\_create\_fargate\_pod\_execution\_role) | Controls if the EKS Fargate pod execution IAM role should be created. | `bool` | `true` | no | diff --git a/main.tf b/main.tf index c5cc718e49..328f279dc7 100644 --- a/main.tf +++ b/main.tf @@ -47,6 +47,7 @@ resource "aws_eks_cluster" "this" { timeouts { create = var.cluster_create_timeout delete = var.cluster_delete_timeout + update = var.cluster_update_timeout } depends_on = [ diff --git a/variables.tf b/variables.tf index 6454e6bad7..a5d9adefe3 100644 --- a/variables.tf +++ b/variables.tf @@ -234,6 +234,12 @@ variable "cluster_delete_timeout" { default = "15m" } +variable "cluster_update_timeout" { + description = "Timeout value when updating the EKS cluster." + type = string + default = "60m" +} + variable "cluster_create_security_group" { description = "Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`." type = bool From 284b038bfda29800d2203d1fce939204c944fd70 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Fri, 17 Sep 2021 16:47:30 +0200 Subject: [PATCH 391/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0638673a2..edfa4df99e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.20.0] - 2021-09-17 +FEATURES: +- Ability to specify cluster update timeout ([#1588](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1588)) + + ## [v17.19.0] - 2021-09-16 REFACTORS: @@ -516,7 +522,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.19.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.20.0...HEAD +[v17.20.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.19.0...v17.20.0 [v17.19.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.18.0...v17.19.0 [v17.18.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.17.0...v17.18.0 [v17.17.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.16.0...v17.17.0 From a06e4c84a9f2dd6768a58b67154dce399270bb1d Mon Sep 17 00:00:00 2001 From: Pascal Bourdier Date: Wed, 22 Sep 2021 15:45:12 +0200 Subject: [PATCH 392/962] docs: Remove `asg_recreate_on_change` in faq (#1596) --- docs/faq.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 3b9e118f6b..5abcc3eb84 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -97,8 +97,6 @@ You are using the cluster autoscaler: - Repeat until all old nodes are drained - Cluster autoscaler will terminate the old nodes after 10-60 minutes automatically -Alternatively you can set the `asg_recreate_on_change = true` worker group option to get the ASG recreated after changes to the launch configuration or template. But be aware of the risks to cluster stability mentioned above. - You can also use a 3rd party tool like Gruntwork's kubergrunt. See the [`eks deploy`](https://github.com/gruntwork-io/kubergrunt#deploy) subcommand. ## How do I create kubernetes resources when creating the cluster? From 13cb55528c3d931f3aefb092af9f511fe6abd29c Mon Sep 17 00:00:00 2001 From: Pascal Bourdier Date: Fri, 24 Sep 2021 12:47:28 +0200 Subject: [PATCH 393/962] fix: Fixed launch_templates_with_managed_node_group example (#1599) --- .../disk_encryption_policy.tf | 1 + .../launchtemplate.tf | 2 -- .../launch_templates_with_managed_node_groups/main.tf | 2 ++ .../variables.tf | 8 ++++---- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf index 0362193505..3f834ad100 100644 --- a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf +++ b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf @@ -2,6 +2,7 @@ resource "aws_iam_service_linked_role" "autoscaling" { aws_service_name = "autoscaling.amazonaws.com" description = "Default Service-Linked Role enables access to AWS Services and Resources used or managed by Auto Scaling" + custom_suffix = "lt_with_managed_node_groups" # the full name is "AWSServiceRoleForAutoScaling_lt_with_managed_node_groups" < 64 characters } #data "aws_caller_identity" "current" {} diff --git a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf index a393770f76..a2840ebc77 100644 --- a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf +++ b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf @@ -37,8 +37,6 @@ resource "aws_launch_template" "default" { } } - instance_type = var.instance_type - monitoring { enabled = true } diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf index 78c2b31217..833b4b9f29 100644 --- a/examples/launch_templates_with_managed_node_groups/main.tf +++ b/examples/launch_templates_with_managed_node_groups/main.tf @@ -62,6 +62,8 @@ module "eks" { launch_template_id = aws_launch_template.default.id launch_template_version = aws_launch_template.default.default_version + instance_types = var.instance_types + additional_tags = { CustomTag = "EKS example" } diff --git a/examples/launch_templates_with_managed_node_groups/variables.tf b/examples/launch_templates_with_managed_node_groups/variables.tf index 351ffdb9cf..9bd936c6f8 100644 --- a/examples/launch_templates_with_managed_node_groups/variables.tf +++ b/examples/launch_templates_with_managed_node_groups/variables.tf @@ -1,6 +1,6 @@ -variable "instance_type" { - description = "Instance type" +variable "instance_types" { + description = "Instance types" # Smallest recommended, where ~1.1Gb of 2Gb memory is available for the Kubernetes pods after ‘warming up’ Docker, Kubelet, and OS - type = string - default = "t3.small" + type = list(string) + default = ["t3.small"] } From 7371db1a51d31263468c6824578e2fb792cc0731 Mon Sep 17 00:00:00 2001 From: Gabriel Virga Date: Tue, 28 Sep 2021 03:39:07 -0400 Subject: [PATCH 394/962] docs: Fix broken URL in README (#1602) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5562a605f1..1e5399e672 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ There is also a [complete example](https://github.com/terraform-aws-modules/terr Root module calls these modules which can also be used separately to create independent resources: -- [fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/fargate) - creates Fargate profiles, see [examples/fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargat) for detailed examples. +- [fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/fargate) - creates Fargate profiles, see [examples/fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate) for detailed examples. From 5ce72fa3b2485972f804fd8632ee184dc54a2553 Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Tue, 28 Sep 2021 11:15:11 +0200 Subject: [PATCH 395/962] docs: update faq about endpoints (#1603) --- docs/faq.md | 20 ++++++++++++++------ docs/spot-instances.md | 2 +- docs/upgrades.md | 5 +++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 5abcc3eb84..b1b31e285d 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -12,15 +12,20 @@ Please open Issues or PRs if you think something is missing. Often caused by a networking or endpoint configuration issue. -At least one of the cluster public or private endpoints must be enabled in order for access to the cluster to work. +At least one of the cluster public or private endpoints must be enabled to access the cluster to work. If you require a public endpoint, setting up both (public and private) and restricting the public endpoint via setting `cluster_endpoint_public_access_cidrs` is recommended. More about communication with an endpoint is available [here](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html). -Nodes need to be able to contact the EKS cluster endpoint. By default the module only creates a public endpoint. To access this endpoint the nodes need outgoing internet access: -- Nodes in private subnets: via a NAT gateway or instance. This will need adding along with appropriate routing rules. +Nodes need to be able to contact the EKS cluster endpoint. By default, the module only creates a public endpoint. To access endpoint, the nodes need outgoing internet access: + +- Nodes in private subnets: via a NAT gateway or instance. It will need adding along with appropriate routing rules. - Nodes in public subnets: assign public IPs to nodes. Set `public_ip = true` in the `worker_groups` list on this module. +> Important: +> If you apply only the public endpoint and setup `cluster_endpoint_public_access_cidrs` to restrict access, remember that EKS nodes also use the public endpoint, so you must allow access to the endpoint. If not, then your nodes will not be working correctly. + Cluster private endpoint can also be enabled by setting `cluster_endpoint_private_access = true` on this module. Node calls to the endpoint stay within the VPC. When the private endpoint is enabled ensure that VPC DNS resolution and hostnames are also enabled: + - If managing the VPC with Terraform: set `enable_dns_hostnames = true` and `enable_dns_support = true` on the `aws_vpc` resource. The [`terraform-aws-module/vpc/aws`](https://github.com/terraform-aws-modules/terraform-aws-vpc/) community module also has these variables. - Otherwise refer to the [AWS VPC docs](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html#vpc-dns-updating) and [AWS EKS Cluster Endpoint Access docs](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html) for more information. @@ -84,6 +89,7 @@ By default the ASG is not configured to be recreated when the launch configurati You need to use a process to drain and cycle the workers. You are not using the cluster autoscaler: + - Add a new instance - Drain an old node `kubectl drain --force --ignore-daemonsets --delete-local-data ip-xxxxxxx.eu-west-1.compute.internal` - Wait for pods to be Running @@ -91,6 +97,7 @@ You are not using the cluster autoscaler: - Repeat the drain and delete process until all old nodes are replaced You are using the cluster autoscaler: + - Drain an old node `kubectl drain --force --ignore-daemonsets --delete-local-data ip-xxxxxxx.eu-west-1.compute.internal` - Wait for pods to be Running - Cluster autoscaler will create new nodes when required @@ -102,6 +109,7 @@ You can also use a 3rd party tool like Gruntwork's kubergrunt. See the [`eks dep ## How do I create kubernetes resources when creating the cluster? You do not need to do anything extra since v12.1.0 of the module as long as the following conditions are met: + - `manage_aws_auth = true` on the module (default) - the kubernetes provider is correctly configured like in the [Usage Example](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/README.md#usage-example). Primarily the module's `cluster_id` output is used as input to the `aws_eks_cluster*` data sources. @@ -114,6 +122,7 @@ You are attempting to use a Terraform 0.12 module with Terraform 0.11. We highly recommend that you upgrade your EKS Terraform config to 0.12 to take advantage of new features in the module. Alternatively you can lock your module to a compatible version if you must stay with terraform 0.11: + ```hcl module "eks" { source = "terraform-aws-modules/eks/aws" @@ -134,7 +143,7 @@ Amazon EKS clusters must contain one or more Linux worker nodes to run core syst 1. Build AWS EKS cluster with the next workers configuration (default Linux): -``` +```hcl worker_groups = [ { name = "worker-group-linux" @@ -146,10 +155,9 @@ worker_groups = [ ``` 2. Apply commands from https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support (use tab with name `Windows`) - 3. Add one more worker group for Windows with required field `platform = "windows"` and update your cluster. Worker group example: -``` +```hcl worker_groups = [ { name = "worker-group-linux" diff --git a/docs/spot-instances.md b/docs/spot-instances.md index f140fe55ea..be89f52f87 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -4,7 +4,7 @@ Spot instances usually cost around 30-70% less than an on-demand instance. So us You need to install a daemonset to catch the 2 minute warning before termination. This will ensure the node is gracefully drained before termination. You can install the [k8s-spot-termination-handler](https://github.com/kube-aws/kube-spot-termination-notice-handler) for this. There's a [Helm chart](https://github.com/helm/charts/tree/master/stable/k8s-spot-termination-handler): -``` +```shell helm install stable/k8s-spot-termination-handler --namespace kube-system ``` diff --git a/docs/upgrades.md b/docs/upgrades.md index 88d29ae1bf..d078bf9e6a 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -6,11 +6,14 @@ In this release, we now decided to remove random_pet resources in Managed Node G 1. Run `terraform apply` with the module version v16.2.0 2. Get your worker group names + ```shell ~ terraform state show 'module.eks.module.node_groups.aws_eks_node_group.workers["example"]' | grep node_group_name node_group_name = "test-eks-mwIwsvui-example-sincere-squid" ``` + 3. Upgrade your module and configure your node groups to use existing names + ```hcl module "eks" { source = "terraform-aws-modules/eks/aws" @@ -30,6 +33,7 @@ module "eks" { # ... } ``` + 4. Run `terraform plan`, you shoud see that only `random_pets` will be destroyed ```shell @@ -55,6 +59,7 @@ Terraform will perform the following actions: Plan: 0 to add, 0 to change, 1 to destroy. ``` + 5. If everything sounds good to you, run `terraform apply` After the first apply, we recommand you to create a new node group and let the module use the `node_group_name_prefix` (by removing the `name` argument) to generate names and avoid collision during node groups re-creation if needed, because the lifce cycle is `create_before_destroy = true`. From 253f927465ccac36bcadefcfb1fd0e39b6244d73 Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Tue, 28 Sep 2021 12:18:43 +0200 Subject: [PATCH 396/962] docs: Updated faq about desired count of instances in node and worker groups (#1604) --- docs/faq.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index b1b31e285d..9fc55cc28f 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -72,15 +72,17 @@ You've added new worker groups. Deleting worker groups from earlier in the list The safest and easiest option is to set `asg_min_size` and `asg_max_size` to 0 on the worker groups to "remove". -## Why does changing the worker group's desired count not do anything? +## Why does changing the node or worker group's desired count not do anything? -The module is configured to ignore this value. Unfortunately Terraform does not support variables within the `lifecycle` block. +The module is configured to ignore this value. Unfortunately, Terraform does not support variables within the `lifecycle` block. -The setting is ignored to allow the cluster autoscaler to work correctly and so that terraform apply does not accidentally remove running workers. +The setting is ignored to allow the cluster autoscaler to work correctly so that `terraform apply` does not accidentally remove running workers. You can change the desired count via the CLI or console if you're not using the cluster autoscaler. -If you are not using autoscaling and really want to control the number of nodes via terraform then set the `asg_min_size` and `asg_max_size` instead. AWS will remove a random instance when you scale down. You will have to weigh the risks here. +If you are not using autoscaling and want to control the number of nodes via terraform, set the `min_capacity` and `max_capacity` for node groups or `asg_min_size` and `asg_max_size` for worker groups. Before changing those values, you must satisfy AWS `desired` capacity constraints (which must be between new min/max values). + +When you scale down AWS will remove a random instance, so you will have to weigh the risks here. ## Why are nodes not recreated when the `launch_configuration`/`launch_template` is recreated? From 1d1a73e138683373eea90525884c9abf0d57bbbb Mon Sep 17 00:00:00 2001 From: better-think Date: Thu, 30 Sep 2021 21:51:52 +0300 Subject: [PATCH 397/962] docs: Updated iam-permissions.md (#1612) --- docs/iam-permissions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index 7c6d19c26e..5285b68094 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -127,6 +127,7 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "iam:TagOpenIDConnectProvider", "iam:TagRole", "iam:UntagRole", + "iam:TagPolicy", "iam:UpdateAssumeRolePolicy", // Following permissions are needed if cluster_enabled_log_types is enabled "logs:CreateLogGroup", From 7f13ffd121ddde78498c7e84ff622be15fe9f7f2 Mon Sep 17 00:00:00 2001 From: better-think Date: Fri, 1 Oct 2021 14:29:51 +0300 Subject: [PATCH 398/962] docs: Update iam-permissions.md (#1613) --- docs/iam-permissions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md index 5285b68094..e6a867a9ec 100644 --- a/docs/iam-permissions.md +++ b/docs/iam-permissions.md @@ -128,6 +128,7 @@ Following IAM permissions are the minimum permissions needed for your IAM user o "iam:TagRole", "iam:UntagRole", "iam:TagPolicy", + "iam:TagInstanceProfile", "iam:UpdateAssumeRolePolicy", // Following permissions are needed if cluster_enabled_log_types is enabled "logs:CreateLogGroup", From f37e5af88adfb86214d8c09424913067bac58a4f Mon Sep 17 00:00:00 2001 From: Paul B Schroeder Date: Thu, 7 Oct 2021 07:21:13 -0500 Subject: [PATCH 399/962] feat: Allow interface_type to be specified in worker_groups_launch_template (#1439) --- locals.tf | 1 + workers_launch_template.tf | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/locals.tf b/locals.tf index 60472a6058..9bebfd03eb 100644 --- a/locals.tf +++ b/locals.tf @@ -94,6 +94,7 @@ locals { launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. root_encrypted = false # Whether the volume should be encrypted or not eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) + interface_type = null # The type of network interface. To create an Elastic Fabric Adapter (EFA), specify 'efa'. cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs market_type = null metadata_http_endpoint = "enabled" # The state of the metadata service: enabled, disabled. diff --git a/workers_launch_template.tf b/workers_launch_template.tf index b675304010..d713cf412d 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -314,6 +314,11 @@ resource "aws_launch_template" "workers_launch_template" { "eni_delete", local.workers_group_defaults["eni_delete"], ) + interface_type = lookup( + var.worker_groups_launch_template[count.index], + "interface_type", + local.workers_group_defaults["interface_type"], + ) security_groups = flatten([ local.worker_security_group_id, var.worker_additional_security_group_ids, From 8866569d53fc244e9765b49b1f8e73866dc87d8c Mon Sep 17 00:00:00 2001 From: omjadas Date: Thu, 7 Oct 2021 23:26:53 +1100 Subject: [PATCH 400/962] feat: Allow snapshot_id to be specified for additional_ebs_volumes (#1431) --- locals.tf | 3 ++- workers.tf | 5 +++++ workers_launch_template.tf | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/locals.tf b/locals.tf index 9bebfd03eb..09dd589214 100644 --- a/locals.tf +++ b/locals.tf @@ -78,10 +78,11 @@ locals { service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. platform = var.default_platform # Platform of workers. Either "linux" or "windows". - additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, throughput, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults + additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, throughput, encrypted, kms_key_id (only on launch-template), delete_on_termination, snapshot_id. Optional values are grabbed from root volume or from defaults additional_instance_store_volumes = [] # A list of additional instance store (local disk) volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), virtual_name. warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. timeouts = {} # A map of timeouts for create/update/delete operations + snapshot_id = null # A custom snapshot ID. # Settings for launch templates root_block_device_name = concat(data.aws_ami.eks_worker.*.root_device_name, [""])[0] # Root device name for Linux workers. If not provided, will assume default Linux AMI was used. diff --git a/workers.tf b/workers.tf index f2cd7cbf1b..8dc2754e56 100644 --- a/workers.tf +++ b/workers.tf @@ -344,6 +344,11 @@ resource "aws_launch_configuration" "workers" { "encrypted", local.workers_group_defaults["root_encrypted"], ) + snapshot_id = lookup( + block_device_mappings.value, + "snapshot_id", + local.workers_group_defaults["snapshot_id"], + ) delete_on_termination = lookup(ebs_block_device.value, "delete_on_termination", true) } } diff --git a/workers_launch_template.tf b/workers_launch_template.tf index d713cf412d..df2928dc5d 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -524,6 +524,11 @@ resource "aws_launch_template" "workers_launch_template" { "kms_key_id", local.workers_group_defaults["root_kms_key_id"], ) + snapshot_id = lookup( + block_device_mappings.value, + "snapshot_id", + local.workers_group_defaults["snapshot_id"], + ) delete_on_termination = lookup(block_device_mappings.value, "delete_on_termination", true) } } From b177806614722ccbf020cc834eac9495714b29d5 Mon Sep 17 00:00:00 2001 From: Junaid Ali Date: Thu, 7 Oct 2021 14:19:23 +0100 Subject: [PATCH 401/962] feat: Enable throughput & iops configs for managed node_groups (#1584) --- examples/managed_node_groups/main.tf | 7 +++++++ modules/node_groups/README.md | 4 +++- modules/node_groups/launch_template.tf | 2 ++ modules/node_groups/locals.tf | 2 ++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index 826223916f..99cc1fd730 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -73,10 +73,17 @@ module "eks" { node_groups = { example = { + create_launch_template = true + desired_capacity = 1 max_capacity = 10 min_capacity = 1 + disk_size = 50 + disk_type = "gp3" + disk_throughput = 150 + disk_iops = 3000 + instance_types = ["t3.large"] capacity_type = "SPOT" k8s_labels = { diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index f004fd19b4..e16b27da7a 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -25,7 +25,9 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | disk\_encrypted | Whether the root disk will be encrypyted. Requires `create_launch_template` to be `true` and `disk_kms_key_id` to be set | bool | false | | disk\_kms\_key\_id | KMS Key used to encrypt the root disk. Requires both `create_launch_template` and `disk_encrypted` to be `true` | string | "" | | disk\_size | Workers' disk size | number | Provider default behavior | -| disk\_type | Workers' disk type. Require `create_launch_template` to be `true`| number | `gp3` | +| disk\_type | Workers' disk type. Require `create_launch_template` to be `true`| string | Provider default behavior | +| disk\_throughput | Workers' disk throughput. Require `create_launch_template` to be `true` and `disk_type` to be `gp3`| number | Provider default behavior | +| disk\_iops | Workers' disk IOPS. Require `create_launch_template` to be `true` and `disk_type` to be `gp3`| number | Provider default behavior | | ebs\_optimized | Enables/disables EBS optimization. Require `create_launch_template` to be `true` | bool | `true` if defined `instance\_types` are not present in `var.ebs\_optimized\_not\_supported` | | enable_monitoring | Enables/disables detailed monitoring. Require `create_launch_template` to be `true`| bool | `true` | | eni_delete | Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) | bool | `true` | diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 2cf7cebe27..0fcc11ebd6 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -37,6 +37,8 @@ resource "aws_launch_template" "workers" { ebs { volume_size = lookup(each.value, "disk_size", null) volume_type = lookup(each.value, "disk_type", null) + iops = lookup(each.value, "disk_iops", null) + throughput = lookup(each.value, "disk_throughput", null) encrypted = lookup(each.value, "disk_encrypted", null) kms_key_id = lookup(each.value, "disk_kms_key_id", null) delete_on_termination = true diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 5951b83fc6..b5ec22d28e 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -16,6 +16,8 @@ locals { kubelet_extra_args = var.workers_group_defaults["kubelet_extra_args"] disk_size = var.workers_group_defaults["root_volume_size"] disk_type = var.workers_group_defaults["root_volume_type"] + disk_iops = var.workers_group_defaults["root_iops"] + disk_throughput = var.workers_group_defaults["root_volume_throughput"] disk_encrypted = var.workers_group_defaults["root_encrypted"] disk_kms_key_id = var.workers_group_defaults["root_kms_key_id"] enable_monitoring = var.workers_group_defaults["enable_monitoring"] From f198efd2c4ed695c8018ee343701454069806790 Mon Sep 17 00:00:00 2001 From: Steve Hipwell Date: Fri, 8 Oct 2021 15:41:10 +0100 Subject: [PATCH 402/962] feat: Fix custom AMI bootstrap (#1580) --- modules/node_groups/README.md | 4 +++- modules/node_groups/launch_template.tf | 9 ++++++-- modules/node_groups/locals.tf | 5 +++++ modules/node_groups/templates/userdata.sh.tpl | 21 ++++++++++++++----- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index e16b27da7a..b7c20cad04 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -18,7 +18,8 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | additional\_tags | Additional tags to apply to node group | map(string) | Only `var.tags` applied | | ami\_release\_version | AMI version of workers | string | Provider default behavior | | ami\_type | AMI Type. See Terraform or AWS docs | string | Provider default behavior | -| ami\_id | ID of custom AMI. If you use a custom AMI, you need to supply bootstrap script via user-data or as AMI built-in. | string | Provider default behavior | +| ami\_id | ID of custom AMI. If you use a custom AMI, you need to set `ami_is_eks_optimized` | string | Provider default behavior | +| ami\_is\_eks\_optimized | If the custom AMI is an EKS optimised image, ignored if `ami_id` is not set. If this is `true` then `bootstrap.sh` is called automatically (max pod logic needs to be manually set), if this is `false` you need to provide all the node configuration in `pre_userdata` | bool | `true` | | capacity\_type | Type of instance capacity to provision. Options are `ON_DEMAND` and `SPOT` | string | Provider default behavior | | create_launch_template | Create and use a default launch template | bool | `false` | | desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` | @@ -83,6 +84,7 @@ No modules. |------|------| | [aws_eks_node_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_launch_template.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_eks_cluster.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | | [cloudinit_config.workers_userdata](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | ## Inputs diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 0fcc11ebd6..0bbb0f8d4e 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -9,10 +9,15 @@ data "cloudinit_config" "workers_userdata" { content_type = "text/x-shellscript" content = templatefile("${path.module}/templates/userdata.sh.tpl", { - pre_userdata = each.value["pre_userdata"] kubelet_extra_args = each.value["kubelet_extra_args"] + pre_userdata = each.value["pre_userdata"] + ami_id = lookup(each.value, "ami_id", "") + ami_is_eks_optimized = each.value["ami_is_eks_optimized"] cluster_name = var.cluster_name - run_bootstrap_script = lookup(each.value, "ami_id", null) != null + cluster_endpoint = data.aws_eks_cluster.default.endpoint + cluster_ca = data.aws_eks_cluster.default.certificate_authority[0].data + capacity_type = lookup(each.value, "capacity_type", "ON_DEMAND") + append_labels = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : "" } ) } diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index b5ec22d28e..8454e0b607 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -1,3 +1,7 @@ +data "aws_eks_cluster" "default" { + name = var.cluster_name +} + locals { # Merge defaults and per-group values to make code cleaner node_groups_expanded = { for k, v in var.node_groups : k => merge( @@ -32,6 +36,7 @@ locals { metadata_http_endpoint = var.workers_group_defaults["metadata_http_endpoint"] metadata_http_tokens = var.workers_group_defaults["metadata_http_tokens"] metadata_http_put_response_hop_limit = var.workers_group_defaults["metadata_http_put_response_hop_limit"] + ami_is_eks_optimized = true }, var.node_groups_defaults, v, diff --git a/modules/node_groups/templates/userdata.sh.tpl b/modules/node_groups/templates/userdata.sh.tpl index 097c1168f8..aab0c5ad79 100644 --- a/modules/node_groups/templates/userdata.sh.tpl +++ b/modules/node_groups/templates/userdata.sh.tpl @@ -1,9 +1,20 @@ #!/bin/bash -e +%{ if length(ami_id) == 0 ~} -# Allow user supplied pre userdata code +# Set variables directly into bootstrap.sh for default AMI +sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" ${kubelet_extra_args}"' /etc/eks/bootstrap.sh +%{else ~} + +# Set variables for custom AMI +API_SERVER_URL=${cluster_endpoint} +B64_CLUSTER_CA=${cluster_ca} +KUBELET_EXTRA_ARGS='--node-labels=eks.amazonaws.com/nodegroup-image=${ami_id},eks.amazonaws.com/capacityType=${capacity_type}${append_labels} ${kubelet_extra_args}' +%{endif ~} + +# User supplied pre userdata ${pre_userdata} +%{ if length(ami_id) > 0 && ami_is_eks_optimized ~} -sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" ${kubelet_extra_args}"' /etc/eks/bootstrap.sh -%{ if run_bootstrap_script } - /etc/eks/bootstrap.sh ${cluster_name} -%{ endif } +# Call bootstrap for EKS optimised custom AMI +/etc/eks/bootstrap.sh ${cluster_name} --apiserver-endpoint "$${API_SERVER_URL}" --b64-cluster-ca "$${B64_CLUSTER_CA}" --kubelet-extra-args "$${KUBELET_EXTRA_ARGS}" +%{ endif ~} From 54a5f1e42b7e2b17f298e4ff19a4821e3de97201 Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Tue, 12 Oct 2021 12:45:41 +0200 Subject: [PATCH 403/962] fix: bug with data source in managed groups submodule (#1633) --- modules/node_groups/launch_template.tf | 4 ++-- modules/node_groups/locals.tf | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 0bbb0f8d4e..5c2243d4c5 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -14,8 +14,8 @@ data "cloudinit_config" "workers_userdata" { ami_id = lookup(each.value, "ami_id", "") ami_is_eks_optimized = each.value["ami_is_eks_optimized"] cluster_name = var.cluster_name - cluster_endpoint = data.aws_eks_cluster.default.endpoint - cluster_ca = data.aws_eks_cluster.default.certificate_authority[0].data + cluster_endpoint = data.aws_eks_cluster.default[0].endpoint + cluster_ca = data.aws_eks_cluster.default[0].certificate_authority[0].data capacity_type = lookup(each.value, "capacity_type", "ON_DEMAND") append_labels = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : "" } diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 8454e0b607..490a42d306 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -1,4 +1,6 @@ data "aws_eks_cluster" "default" { + count = var.create_eks ? 1 : 0 + name = var.cluster_name } From 99d289988df56cd8469a2e17f2bfb220dc87b701 Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Tue, 12 Oct 2021 15:20:14 +0200 Subject: [PATCH 404/962] fix: Rebuild examples (#1625) --- README.md | 14 +- examples/_bootstrap/main.tf | 50 ----- examples/_bootstrap/outputs.tf | 14 -- examples/_bootstrap/variables.tf | 0 examples/_bootstrap/versions.tf | 9 - examples/bottlerocket/README.md | 12 +- examples/bottlerocket/main.tf | 102 ++++++++-- examples/bottlerocket/outputs.tf | 5 - examples/bottlerocket/versions.tf | 8 +- examples/complete/README.md | 9 +- examples/complete/main.tf | 94 ++++++--- examples/complete/outputs.tf | 5 - examples/complete/variables.tf | 1 - examples/complete/versions.tf | 2 +- examples/fargate/README.md | 14 +- examples/fargate/main.tf | 155 ++++++++------- examples/fargate/versions.tf | 2 +- examples/instance_refresh/README.md | 82 ++++++++ examples/instance_refresh/main.tf | 160 +++++++++------ examples/instance_refresh/variables.tf | 17 -- examples/instance_refresh/versions.tf | 4 +- examples/irsa/README.md | 93 ++++----- .../irsa/cluster-autoscaler-chart-values.yaml | 14 -- examples/irsa/irsa.tf | 58 +++++- examples/irsa/locals.tf | 5 - examples/irsa/main.tf | 120 ++++++++---- examples/irsa/versions.tf | 3 +- .../README.md | 19 +- examples/launch_templates/main.tf | 119 +++++++---- examples/launch_templates/outputs.tf | 38 ++-- examples/launch_templates/pre_userdata.sh | 2 +- examples/launch_templates/versions.tf | 3 +- .../README.md | 70 +++++++ .../launchtemplate.tf | 12 +- .../main.tf | 139 +++++++++---- .../variables.tf | 6 - .../versions.tf | 2 +- examples/managed_node_groups/README.md | 73 +++++++ examples/managed_node_groups/main.tf | 184 ++++++++++-------- examples/managed_node_groups/versions.tf | 4 +- examples/secrets_encryption/README.md | 66 +++++++ examples/secrets_encryption/main.tf | 126 +++++++----- examples/secrets_encryption/variables.tf | 48 ----- examples/secrets_encryption/versions.tf | 2 +- variables.tf | 6 +- 45 files changed, 1272 insertions(+), 699 deletions(-) delete mode 100644 examples/_bootstrap/main.tf delete mode 100644 examples/_bootstrap/outputs.tf delete mode 100644 examples/_bootstrap/variables.tf delete mode 100644 examples/_bootstrap/versions.tf create mode 100644 examples/instance_refresh/README.md delete mode 100644 examples/irsa/cluster-autoscaler-chart-values.yaml delete mode 100644 examples/irsa/locals.tf rename examples/{_bootstrap => launch_templates}/README.md (53%) create mode 100644 examples/launch_templates_with_managed_node_groups/README.md create mode 100644 examples/managed_node_groups/README.md create mode 100644 examples/secrets_encryption/README.md diff --git a/README.md b/README.md index 1e5399e672..ab4fe8b1ec 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,6 @@ Terraform module which creates Kubernetes cluster resources on AWS EKS. - Support AWS EKS Optimized or Custom AMI - Create or manage security groups that allow communication and coordination - ## Important note Kubernetes is evolving a lot, and each minor version includes new features, fixes, or changes. @@ -24,7 +23,6 @@ Kubernetes is evolving a lot, and each minor version includes new features, fixe You also need to ensure that your applications and add ons are updated, or workloads could fail after the upgrade is complete. For action, you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). - ## Usage example ```hcl @@ -61,7 +59,6 @@ module "eks" { There is also a [complete example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete) which shows large set of features available in the module. - ## Submodules Root module calls these modules which can also be used separately to create independent resources: @@ -71,14 +68,12 @@ Root module calls these modules which can also be used separately to create inde - [node_groups](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/node_groups) - creates Managed Node Group resources --> - ## Notes - By default, this module manages the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready. We implemented a "retry" logic with a [fork of the http provider](https://github.com/terraform-aws-modules/terraform-provider-http). This fork adds the support of a self-signed CA certificate. The original PR can be found [here](https://github.com/hashicorp/terraform-provider-http/pull/29). - Setting `instance_refresh_enabled = true` will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. Find the complete example here [instance_refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/instance_refresh). - ## Documentation ### Official docs @@ -93,7 +88,6 @@ Root module calls these modules which can also be used separately to create inde - [IAM Permissions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md): Minimum IAM permissions needed to setup EKS Cluster. - [FAQ](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md): Frequently Asked Questions - ## Examples There are detailed examples available for you to see how certain features of this module can be used in a straightforward way. Make sure to check them and run them before opening an issue. [Here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md) you can find the list of the minimum IAM Permissions required to create EKS cluster. @@ -102,14 +96,12 @@ There are detailed examples available for you to see how certain features of thi - [Bottlerocket](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/bottlerocket) - Create EKS cluster using [Bottlerocket AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami-bottlerocket.html). - [Fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate) - Create EKS cluster with [Fargate profiles](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) and attach Fargate profiles to an existing EKS cluster. - ## Contributing Report issues/questions/feature requests on in the [issues](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/new) section. Full contributing [guidelines are covered here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/.github/CONTRIBUTING.md). - ## Authors This module has been originally created by [Brandon O'Connor](https://github.com/brandoconnor), and was maintained by [Max Williams](https://github.com/max-rocket-internet), [Thierno IB. BARRY](https://github.com/barryib) and many more [contributors listed here](https://github.com/terraform-aws-modules/terraform-aws-eks/graphs/contributors)! @@ -243,9 +235,9 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [manage\_aws\_auth](#input\_manage\_aws\_auth) | Whether to apply the aws-auth configmap file. | `bool` | `true` | no | | [manage\_cluster\_iam\_resources](#input\_manage\_cluster\_iam\_resources) | Whether to let the module manage cluster IAM resources. If set to false, cluster\_iam\_role\_name must be specified. | `bool` | `true` | no | | [manage\_worker\_iam\_resources](#input\_manage\_worker\_iam\_resources) | Whether to let the module manage worker IAM resources. If set to false, iam\_instance\_profile\_name must be specified for workers. | `bool` | `true` | no | -| [map\_accounts](#input\_map\_accounts) | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | `list(string)` | `[]` | no | -| [map\_roles](#input\_map\_roles) | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
| `[]` | no | -| [map\_users](#input\_map\_users) | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | +| [map\_accounts](#input\_map\_accounts) | Additional AWS account numbers to add to the aws-auth configmap. | `list(string)` | `[]` | no | +| [map\_roles](#input\_map\_roles) | Additional IAM roles to add to the aws-auth configmap. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
| `[]` | no | +| [map\_users](#input\_map\_users) | Additional IAM users to add to the aws-auth configmap. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | | [node\_groups](#input\_node\_groups) | Map of map of node groups to create. See `node_groups` module's documentation for more details | `any` | `{}` | no | | [node\_groups\_defaults](#input\_node\_groups\_defaults) | Map of values to be applied to all node groups. See `node_groups` module's documentation for more details | `any` | `{}` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider. | `list(string)` | `[]` | no | diff --git a/examples/_bootstrap/main.tf b/examples/_bootstrap/main.tf deleted file mode 100644 index 8492741f6b..0000000000 --- a/examples/_bootstrap/main.tf +++ /dev/null @@ -1,50 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - region = "eu-west-1" - name = "bootstrap-example" - vpc_cidr = "10.0.0.0/16" - - cluster_name = "test-eks-${random_string.suffix.result}" -} - -data "aws_availability_zones" "available" {} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -################################################################################ -# Supporting Resources -################################################################################ - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = "10.0.0.0/16" - - azs = data.aws_availability_zones.available.names - public_subnets = [for k, v in data.aws_availability_zones.available.names : cidrsubnet(local.vpc_cidr, 8, k)] - private_subnets = [for k, v in data.aws_availability_zones.available.names : cidrsubnet(local.vpc_cidr, 8, k + 10)] - - # NAT Gateway is disabled in the examples primarily to save costs and be able to recreate VPC faster. - enable_nat_gateway = false - single_nat_gateway = false - - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } -} diff --git a/examples/_bootstrap/outputs.tf b/examples/_bootstrap/outputs.tf deleted file mode 100644 index 87a2e49621..0000000000 --- a/examples/_bootstrap/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "region" { - description = "AWS region" - value = local.region -} - -output "cluster_name" { - description = "Name of EKS Cluster used in tags for subnets" - value = local.cluster_name -} - -output "vpc" { - description = "Complete output of VPC module" - value = module.vpc -} diff --git a/examples/_bootstrap/variables.tf b/examples/_bootstrap/variables.tf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/_bootstrap/versions.tf b/examples/_bootstrap/versions.tf deleted file mode 100644 index 5539f13af9..0000000000 --- a/examples/_bootstrap/versions.tf +++ /dev/null @@ -1,9 +0,0 @@ -terraform { - required_version = ">= 0.13.1" - - required_providers { - aws = ">= 3.22.0" - random = ">= 2.1" - kubernetes = ">= 1.11" - } -} diff --git a/examples/bottlerocket/README.md b/examples/bottlerocket/README.md index f51a0b6952..e02687da98 100644 --- a/examples/bottlerocket/README.md +++ b/examples/bottlerocket/README.md @@ -1,6 +1,6 @@ # AWS EKS cluster running Bottlerocket AMI -Configuration in this directory creates EKS cluster with nodes running [AWS Bottlerocket OS](https://github.com/bottlerocket-os/bottlerocket) +Configuration in this directory creates EKS cluster with workers group running [AWS Bottlerocket OS](https://github.com/bottlerocket-os/bottlerocket) This is a minimalistic example which shows what knobs to turn to make Bottlerocket work. @@ -25,6 +25,8 @@ Note that this example may create resources which cost money. Run `terraform des |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.22.0 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | | [tls](#requirement\_tls) | >= 2.0 | @@ -41,6 +43,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| | [eks](#module\_eks) | ../.. | | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources @@ -51,8 +54,10 @@ Note that this example may create resources which cost money. Run `terraform des | [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [tls_private_key.nodes](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | | [aws_ami.bottlerocket_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | -| [aws_subnet_ids.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet_ids) | data source | -| [aws_vpc.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs @@ -67,5 +72,4 @@ No inputs. | [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | | [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | | [node\_groups](#output\_node\_groups) | Outputs from node groups | -| [region](#output\_region) | AWS region. | diff --git a/examples/bottlerocket/main.tf b/examples/bottlerocket/main.tf index 9631ef5bc7..bffa83238b 100644 --- a/examples/bottlerocket/main.tf +++ b/examples/bottlerocket/main.tf @@ -3,21 +3,30 @@ provider "aws" { } locals { - region = "eu-west-1" - k8s_version = "1.21" + name = "bottlerocket-${random_string.suffix.result}" + cluster_version = "1.20" + region = "eu-west-1" } +################################################################################ +# EKS Module +################################################################################ + module "eks" { source = "../.." - cluster_name = "bottlerocket-${random_string.suffix.result}" - cluster_version = local.k8s_version + cluster_name = local.name + cluster_version = local.cluster_version - vpc_id = data.aws_vpc.default.id - subnets = data.aws_subnet_ids.default.ids + vpc_id = module.vpc.vpc_id + subnets = [module.vpc.private_subnets[0], module.vpc.public_subnets[1]] + fargate_subnets = [module.vpc.private_subnets[2]] + + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true write_kubeconfig = false - manage_aws_auth = false + manage_aws_auth = true worker_groups_launch_template = [ { @@ -40,7 +49,7 @@ module "eks" { userdata_template_extra_args = { enable_admin_container = false enable_control_container = true - aws_region = local.region + aws_region = data.aws_region.current.name } # example of k8s/kubelet configuration via additional_userdata additional_userdata = < [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.22.0 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -34,7 +34,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.22.0 | -| [terraform](#provider\_terraform) | n/a | +| [random](#provider\_random) | >= 2.1 | ## Modules @@ -44,6 +44,7 @@ Note that this example may create resources which cost money. Run `terraform des | [disabled\_fargate](#module\_disabled\_fargate) | ../../modules/fargate | | | [disabled\_node\_groups](#module\_disabled\_node\_groups) | ../../modules/node_groups | | | [eks](#module\_eks) | ../.. | | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources @@ -52,9 +53,10 @@ Note that this example may create resources which cost money. Run `terraform des | [aws_security_group.all_worker_mgmt](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.worker_group_mgmt_one](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.worker_group_mgmt_two](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | | [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | -| [terraform_remote_state.bootstrap](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | ## Inputs @@ -69,5 +71,4 @@ No inputs. | [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | | [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | | [node\_groups](#output\_node\_groups) | Outputs from node groups | -| [region](#output\_region) | AWS region. | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index c0e2c8b679..ba762e8e03 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -2,15 +2,30 @@ provider "aws" { region = local.region } +locals { + name = "complete-${random_string.suffix.result}" + cluster_version = "1.20" + region = "eu-west-1" +} + +################################################################################ +# EKS Module +################################################################################ + module "eks" { source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.21" + cluster_name = local.name + cluster_version = local.cluster_version + + vpc_id = module.vpc.vpc_id + subnets = [module.vpc.private_subnets[0], module.vpc.public_subnets[1]] + fargate_subnets = [module.vpc.private_subnets[2]] + + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + - vpc_id = local.vpc.vpc_id - subnets = [local.vpc.private_subnets[0], local.vpc.public_subnets[1]] - fargate_subnets = [local.vpc.private_subnets[2]] worker_additional_security_group_ids = [aws_security_group.all_worker_mgmt.id] @@ -130,15 +145,15 @@ module "eks" { ] tags = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" } } -#################### +################################################################################ # Disabled creation -#################### +################################################################################ module "disabled_eks" { source = "../.." @@ -158,9 +173,9 @@ module "disabled_node_groups" { create_eks = false } -############# -# Kubernetes -############# +################################################################################ +# Kubernetes provider configuration +################################################################################ data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id @@ -177,12 +192,12 @@ provider "kubernetes" { } ################################################################################ -# Supporting resources +# Additional security groups for workers ################################################################################ resource "aws_security_group" "worker_group_mgmt_one" { name_prefix = "worker_group_mgmt_one" - vpc_id = local.vpc.vpc_id + vpc_id = module.vpc.vpc_id ingress { from_port = 22 @@ -197,7 +212,7 @@ resource "aws_security_group" "worker_group_mgmt_one" { resource "aws_security_group" "worker_group_mgmt_two" { name_prefix = "worker_group_mgmt_two" - vpc_id = local.vpc.vpc_id + vpc_id = module.vpc.vpc_id ingress { from_port = 22 @@ -212,7 +227,7 @@ resource "aws_security_group" "worker_group_mgmt_two" { resource "aws_security_group" "all_worker_mgmt" { name_prefix = "all_worker_management" - vpc_id = local.vpc.vpc_id + vpc_id = module.vpc.vpc_id ingress { from_port = 22 @@ -227,21 +242,44 @@ resource "aws_security_group" "all_worker_mgmt" { } } - ################################################################################ -# Supporting resources (managed in "_bootstrap" directory) +# Supporting resources ################################################################################ -data "terraform_remote_state" "bootstrap" { - backend = "local" +data "aws_availability_zones" "available" { +} - config = { - path = "../_bootstrap/terraform.tfstate" - } +resource "random_string" "suffix" { + length = 8 + special = false } -locals { - region = data.terraform_remote_state.bootstrap.outputs.region - cluster_name = data.terraform_remote_state.bootstrap.outputs.cluster_name - vpc = data.terraform_remote_state.bootstrap.outputs.vpc +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } } diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index 356410219f..10a3a96604 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -1,8 +1,3 @@ -output "region" { - description = "AWS region." - value = local.region -} - output "cluster_endpoint" { description = "Endpoint for EKS control plane." value = module.eks.cluster_endpoint diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf index 8b13789179..e69de29bb2 100644 --- a/examples/complete/variables.tf +++ b/examples/complete/variables.tf @@ -1 +0,0 @@ - diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index 120d873e2f..bbcf893252 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -5,6 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = ">= 1.11" + kubernetes = "~> 2.0" } } diff --git a/examples/fargate/README.md b/examples/fargate/README.md index 6abd31b524..b4e79abdb7 100644 --- a/examples/fargate/README.md +++ b/examples/fargate/README.md @@ -1,8 +1,9 @@ # AWS EKS Cluster with Fargate profiles Configuration in this directory creates EKS cluster with Fargate profiles in two different ways: + - Using a root module, where EKS Cluster and Fargate profiles should be created at once. This is the default behaviour for most users. -- Using `modules/fargate` submodule where Fargate profiles should be attached to the barebone EKS Cluster. +- Using `modules/fargate` submodule where Fargate profiles should be attached to the existing EKS Cluster. ## Usage @@ -23,7 +24,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.22.0 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -32,25 +33,24 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.22.0 | -| [terraform](#provider\_terraform) | n/a | +| [random](#provider\_random) | >= 2.1 | ## Modules | Name | Source | Version | |------|--------|---------| -| [barebone\_eks](#module\_barebone\_eks) | ../.. | | | [eks](#module\_eks) | ../.. | | | [fargate\_profile\_existing\_cluster](#module\_fargate\_profile\_existing\_cluster) | ../../modules/fargate | | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources | Name | Type | |------|------| -| [aws_eks_cluster.barebone](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | -| [aws_eks_cluster_auth.barebone](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | -| [terraform_remote_state.bootstrap](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | ## Inputs diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf index 0901843445..869ed7e991 100644 --- a/examples/fargate/main.tf +++ b/examples/fargate/main.tf @@ -2,15 +2,50 @@ provider "aws" { region = local.region } +locals { + name = "fargate-${random_string.suffix.result}" + cluster_version = "1.20" + region = "eu-west-1" +} + +################################################################################ +# EKS Module +################################################################################ + module "eks" { source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.21" + cluster_name = local.name + cluster_version = local.cluster_version + + vpc_id = module.vpc.vpc_id + subnets = [module.vpc.private_subnets[0], module.vpc.public_subnets[1]] + fargate_subnets = [module.vpc.private_subnets[2]] + + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true - vpc_id = local.vpc.vpc_id - subnets = [local.vpc.private_subnets[0], local.vpc.public_subnets[1]] - fargate_subnets = [local.vpc.private_subnets[2]] + # You require a node group to schedule coredns which is critical for running correctly internal DNS. + # If you want to use only fargate you must follow docs `(Optional) Update CoreDNS` + # available under https://docs.aws.amazon.com/eks/latest/userguide/fargate-getting-started.html + node_groups = { + example = { + desired_capacity = 1 + + instance_types = ["t3.large"] + k8s_labels = { + Example = "managed_node_groups" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + additional_tags = { + ExtraTag = "example" + } + update_config = { + max_unavailable_percentage = 50 # or set `max_unavailable` + } + } + } fargate_profiles = { default = { @@ -49,7 +84,7 @@ module "eks" { ] # Using specific subnets instead of the ones configured in EKS (`subnets` and `fargate_subnets`) - subnets = [local.vpc.private_subnets[1]] + subnets = [module.vpc.private_subnets[1]] tags = { Owner = "secondary" @@ -60,12 +95,13 @@ module "eks" { manage_aws_auth = false tags = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" } } + ############################################## # Calling submodule with existing EKS cluster ############################################## @@ -73,8 +109,8 @@ module "eks" { module "fargate_profile_existing_cluster" { source = "../../modules/fargate" - cluster_name = module.barebone_eks.cluster_id - subnets = [local.vpc.private_subnets[0], local.vpc.private_subnets[1]] + cluster_name = module.eks.cluster_id + subnets = [module.vpc.private_subnets[0], module.vpc.private_subnets[2]] fargate_profiles = { profile1 = { @@ -95,7 +131,8 @@ module "fargate_profile_existing_cluster" { ] tags = { - Owner = "profile1" + Owner = "profile1" + submodule = "true" } } @@ -111,22 +148,25 @@ module "fargate_profile_existing_cluster" { ] # Using specific subnets instead of the ones configured in EKS (`subnets` and `fargate_subnets`) - subnets = [local.vpc.private_subnets[1]] + subnets = [module.vpc.private_subnets[0]] tags = { - Owner = "profile2" + Owner = "profile2" + submodule = "true" } } } tags = { - DoYouLoveFargate = "Yes" + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" } } -############# -# Kubernetes -############# +################################################################################ +# Kubernetes provider configuration +################################################################################ data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id @@ -142,60 +182,45 @@ provider "kubernetes" { token = data.aws_eks_cluster_auth.cluster.token } -############################################################ -# Barebone EKS Cluster where submodules can add extra stuff -############################################################ - -module "barebone_eks" { - source = "../.." - - cluster_name = "barebone-${local.cluster_name}" - cluster_version = "1.21" - - vpc_id = local.vpc.vpc_id - subnets = local.vpc.private_subnets - - tags = { - Environment = "test" - Barebone = "yes_please" - } -} - -############# -# Kubernetes -############# - -data "aws_eks_cluster" "barebone" { - name = module.barebone_eks.cluster_id -} +################################################################################ +# Supporting Resources +################################################################################ -data "aws_eks_cluster_auth" "barebone" { - name = module.barebone_eks.cluster_id +data "aws_availability_zones" "available" { } -provider "kubernetes" { - alias = "barebone" - - host = data.aws_eks_cluster.barebone.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.barebone.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.barebone.token +resource "random_string" "suffix" { + length = 8 + special = false } +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = "1" + } -################################################################################ -# Supporting resources (managed in "_bootstrap" directory) -################################################################################ - -data "terraform_remote_state" "bootstrap" { - backend = "local" + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } - config = { - path = "../_bootstrap/terraform.tfstate" + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" } } -locals { - region = data.terraform_remote_state.bootstrap.outputs.region - cluster_name = data.terraform_remote_state.bootstrap.outputs.cluster_name - vpc = data.terraform_remote_state.bootstrap.outputs.vpc -} diff --git a/examples/fargate/versions.tf b/examples/fargate/versions.tf index 120d873e2f..bbcf893252 100644 --- a/examples/fargate/versions.tf +++ b/examples/fargate/versions.tf @@ -5,6 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = ">= 1.11" + kubernetes = "~> 2.0" } } diff --git a/examples/instance_refresh/README.md b/examples/instance_refresh/README.md new file mode 100644 index 0000000000..73b836d689 --- /dev/null +++ b/examples/instance_refresh/README.md @@ -0,0 +1,82 @@ +# Instance refresh example + +This is EKS example using [instance refresh](https://aws.amazon.com/blogs/compute/introducing-instance-refresh-for-ec2-auto-scaling/) feature for worker groups. + +See [the official documentation](https://docs.aws.amazon.com/autoscaling/ec2/userguide/asg-instance-refresh.html) for more details. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [helm](#requirement\_helm) | ~> 2.0 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [local](#requirement\_local) | >= 1.4 | +| [random](#requirement\_random) | >= 2.1 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.22.0 | +| [helm](#provider\_helm) | ~> 2.0 | +| [random](#provider\_random) | >= 2.1 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [aws\_node\_termination\_handler\_role](#module\_aws\_node\_termination\_handler\_role) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | 4.1.0 | +| [aws\_node\_termination\_handler\_sqs](#module\_aws\_node\_termination\_handler\_sqs) | terraform-aws-modules/sqs/aws | ~> 3.0.0 | +| [eks](#module\_eks) | ../.. | | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_autoscaling_lifecycle_hook.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_lifecycle_hook) | resource | +| [aws_cloudwatch_event_rule.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_rule.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_target.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_cloudwatch_event_target.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_iam_policy.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [helm_release.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [aws_iam_policy_document.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.aws_node_termination_handler_events](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | +| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | +| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | +| [sqs\_queue\_asg\_notification\_arn](#output\_sqs\_queue\_asg\_notification\_arn) | SQS queue ASG notification ARN | +| [sqs\_queue\_asg\_notification\_url](#output\_sqs\_queue\_asg\_notification\_url) | SQS queue ASG notification URL | + diff --git a/examples/instance_refresh/main.tf b/examples/instance_refresh/main.tf index 54735fcddd..f32964b1d7 100644 --- a/examples/instance_refresh/main.tf +++ b/examples/instance_refresh/main.tf @@ -1,24 +1,18 @@ -# Based on the official aws-node-termination-handler setup guide at https://github.com/aws/aws-node-termination-handler#infrastructure-setup - provider "aws" { region = local.region } -data "aws_caller_identity" "current" {} - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id +locals { + name = "instance_refresh-${random_string.suffix.result}" + cluster_version = "1.20" + region = "eu-west-1" } -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} +################################################################################ +# EKS Module +################################################################################ -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} +# Based on the official aws-node-termination-handler setup guide at https://github.com/aws/aws-node-termination-handler#infrastructure-setup provider "helm" { kubernetes { @@ -28,29 +22,7 @@ provider "helm" { } } -data "aws_availability_zones" "available" { -} - -locals { - cluster_name = "test-refresh-${random_string.suffix.result}" - region = "eu-west-1" -} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0.0" - - name = local.cluster_name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_dns_hostnames = true -} +data "aws_caller_identity" "current" {} data "aws_iam_policy_document" "aws_node_termination_handler" { statement { @@ -84,10 +56,12 @@ data "aws_iam_policy_document" "aws_node_termination_handler" { } resource "aws_iam_policy" "aws_node_termination_handler" { - name = "${local.cluster_name}-aws-node-termination-handler" + name = "${local.name}-aws-node-termination-handler" policy = data.aws_iam_policy_document.aws_node_termination_handler.json } +data "aws_region" "current" {} + data "aws_iam_policy_document" "aws_node_termination_handler_events" { statement { effect = "Allow" @@ -102,7 +76,7 @@ data "aws_iam_policy_document" "aws_node_termination_handler_events" { "sqs:SendMessage", ] resources = [ - "arn:aws:sqs:${local.region}:${data.aws_caller_identity.current.account_id}:${local.cluster_name}", + "arn:aws:sqs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${local.name}", ] } } @@ -110,13 +84,13 @@ data "aws_iam_policy_document" "aws_node_termination_handler_events" { module "aws_node_termination_handler_sqs" { source = "terraform-aws-modules/sqs/aws" version = "~> 3.0.0" - name = local.cluster_name + name = local.name message_retention_seconds = 300 policy = data.aws_iam_policy_document.aws_node_termination_handler_events.json } resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_asg" { - name = "${local.cluster_name}-asg-termination" + name = "${local.name}-asg-termination" description = "Node termination event rule" event_pattern = jsonencode( { @@ -132,13 +106,13 @@ resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_asg" { } resource "aws_cloudwatch_event_target" "aws_node_termination_handler_asg" { - target_id = "${local.cluster_name}-asg-termination" + target_id = "${local.name}-asg-termination" rule = aws_cloudwatch_event_rule.aws_node_termination_handler_asg.name arn = module.aws_node_termination_handler_sqs.sqs_queue_arn } resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_spot" { - name = "${local.cluster_name}-spot-termination" + name = "${local.name}-spot-termination" description = "Node termination event rule" event_pattern = jsonencode( { @@ -154,7 +128,7 @@ resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_spot" { } resource "aws_cloudwatch_event_target" "aws_node_termination_handler_spot" { - target_id = "${local.cluster_name}-spot-termination" + target_id = "${local.name}-spot-termination" rule = aws_cloudwatch_event_rule.aws_node_termination_handler_spot.name arn = module.aws_node_termination_handler_sqs.sqs_queue_arn } @@ -163,11 +137,11 @@ module "aws_node_termination_handler_role" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" version = "4.1.0" create_role = true - role_description = "IRSA role for ANTH, cluster ${local.cluster_name}" - role_name_prefix = local.cluster_name + role_description = "IRSA role for ANTH, cluster ${local.name}" + role_name_prefix = local.name provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") role_policy_arns = [aws_iam_policy.aws_node_termination_handler.arn] - oidc_fully_qualified_subjects = ["system:serviceaccount:${var.namespace}:${var.serviceaccount}"] + oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:aws-node-termination-handler"] } resource "helm_release" "aws_node_termination_handler" { @@ -176,19 +150,19 @@ resource "helm_release" "aws_node_termination_handler" { ] name = "aws-node-termination-handler" - namespace = var.namespace + namespace = "kube-system" repository = "https://aws.github.io/eks-charts" chart = "aws-node-termination-handler" - version = var.aws_node_termination_handler_chart_version + version = "0.15.0" create_namespace = true set { name = "awsRegion" - value = local.region + value = data.aws_region.current.name } set { name = "serviceAccount.name" - value = var.serviceaccount + value = "aws-node-termination-handler" } set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" @@ -226,12 +200,18 @@ resource "aws_autoscaling_lifecycle_hook" "aws_node_termination_handler" { } module "eks" { - source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.20" - subnets = module.vpc.public_subnets - vpc_id = module.vpc.vpc_id - enable_irsa = true + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + + vpc_id = module.vpc.vpc_id + subnets = module.vpc.private_subnets + + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + enable_irsa = true worker_groups_launch_template = [ { name = "refresh" @@ -257,4 +237,70 @@ module "eks" { ] } ] + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# Kubernetes provider configuration +################################################################################ + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} + +################################################################################ +# Supporting Resources +################################################################################ + +data "aws_availability_zones" "available" { +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } } diff --git a/examples/instance_refresh/variables.tf b/examples/instance_refresh/variables.tf index 60b24336b3..e69de29bb2 100644 --- a/examples/instance_refresh/variables.tf +++ b/examples/instance_refresh/variables.tf @@ -1,17 +0,0 @@ -variable "aws_node_termination_handler_chart_version" { - description = "Version of the aws-node-termination-handler Helm chart to install." - type = string - default = "0.15.0" -} - -variable "namespace" { - description = "Namespace for the aws-node-termination-handler." - type = string - default = "kube-system" -} - -variable "serviceaccount" { - description = "Serviceaccount for the aws-node-termination-handler." - type = string - default = "aws-node-termination-handler" -} diff --git a/examples/instance_refresh/versions.tf b/examples/instance_refresh/versions.tf index 67281c8d51..f546ca0cf0 100644 --- a/examples/instance_refresh/versions.tf +++ b/examples/instance_refresh/versions.tf @@ -5,7 +5,7 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = "~> 1.11" - helm = "~> 2.1.2" + kubernetes = "~> 2.0" + helm = "~> 2.0" } } diff --git a/examples/irsa/README.md b/examples/irsa/README.md index ab81f649d7..770ca3f902 100644 --- a/examples/irsa/README.md +++ b/examples/irsa/README.md @@ -2,64 +2,69 @@ This example shows how to create an IAM role to be used for a Kubernetes `ServiceAccount`. It will create a policy and role to be used by the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) using the [public Helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler). -The AWS documentation for IRSA is here: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html +See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) for more details. -## Setup +## Usage -Run Terraform: +To run this example you need to execute: -``` -terraform init -terraform apply +```bash +$ terraform init +$ terraform plan +$ terraform apply ``` -Set kubectl context to the new cluster: `export KUBECONFIG=kubeconfig_test-eks-irsa` +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. -Check that there is a node that is `Ready`: + +## Requirements -``` -$ kubectl get nodes -NAME STATUS ROLES AGE VERSION -ip-10-0-2-190.us-west-2.compute.internal Ready 6m39s v1.14.8-eks-b8860f -``` +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [helm](#requirement\_helm) | ~> 2.0 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [local](#requirement\_local) | >= 1.4 | +| [random](#requirement\_random) | >= 2.1 | -Replace `` with your AWS account ID in `cluster-autoscaler-chart-values.yaml`. There is output from terraform for this. +## Providers -Install the chart using the provided values file: +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.22.0 | +| [helm](#provider\_helm) | ~> 2.0 | +| [random](#provider\_random) | >= 2.1 | -``` -$ helm repo add autoscaler https://kubernetes.github.io/autoscaler -$ helm repo update -$ helm install cluster-autoscaler --namespace kube-system autoscaler/cluster-autoscaler --values cluster-autoscaler-chart-values.yaml -``` +## Modules -## Verify +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | | +| [iam\_assumable\_role\_admin](#module\_iam\_assumable\_role\_admin) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | ~> 4.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | -Ensure the cluster-autoscaler pod is running: +## Resources -``` -$ kubectl --namespace=kube-system get pods -l "app.kubernetes.io/name=aws-cluster-autoscaler-chart" -NAME READY STATUS RESTARTS AGE -cluster-autoscaler-aws-cluster-autoscaler-chart-5545d4b97-9ztpm 1/1 Running 0 3m -``` +| Name | Type | +|------|------| +| [aws_iam_policy.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [helm_release.cluster-autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [aws_iam_policy_document.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | -Observe the `AWS_*` environment variables that were added to the pod automatically by EKS: +## Inputs -``` -kubectl --namespace=kube-system get pods -l "app.kubernetes.io/name=aws-cluster-autoscaler-chart" -o yaml | grep -A3 AWS_ROLE_ARN - -- name: AWS_ROLE_ARN - value: arn:aws:iam::xxxxxxxxx:role/cluster-autoscaler -- name: AWS_WEB_IDENTITY_TOKEN_FILE - value: /var/run/secrets/eks.amazonaws.com/serviceaccount/token -``` +No inputs. -Verify it is working by checking the logs, you should see that it has discovered the autoscaling group successfully: +## Outputs -``` -kubectl --namespace=kube-system logs -l "app.kubernetes.io/name=aws-cluster-autoscaler-chart" - -I0128 14:59:00.901513 1 auto_scaling_groups.go:354] Regenerating instance to ASG map for ASGs: [test-eks-irsa-worker-group-12020012814125354700000000e] -I0128 14:59:00.969875 1 auto_scaling_groups.go:138] Registering ASG test-eks-irsa-worker-group-12020012814125354700000000e -I0128 14:59:00.969906 1 aws_manager.go:263] Refreshed ASG list, next refresh after 2020-01-28 15:00:00.969901767 +0000 UTC m=+61.310501783 -``` +| Name | Description | +|------|-------------| +| [aws\_account\_id](#output\_aws\_account\_id) | IAM AWS account id | + diff --git a/examples/irsa/cluster-autoscaler-chart-values.yaml b/examples/irsa/cluster-autoscaler-chart-values.yaml deleted file mode 100644 index 4e5494de30..0000000000 --- a/examples/irsa/cluster-autoscaler-chart-values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -awsRegion: eu-west-1 - -rbac: - create: true - serviceAccount: - # This value should match local.k8s_service_account_name in locals.tf - name: cluster-autoscaler-aws-cluster-autoscaler-chart - annotations: - # This value should match the ARN of the role created by module.iam_assumable_role_admin in irsa.tf - eks.amazonaws.com/role-arn: "arn:aws:iam:::role/cluster-autoscaler" - -autoDiscovery: - clusterName: test-eks-irsa - enabled: true diff --git a/examples/irsa/irsa.tf b/examples/irsa/irsa.tf index 7bb9f7f258..a36d0e3394 100644 --- a/examples/irsa/irsa.tf +++ b/examples/irsa/irsa.tf @@ -1,6 +1,62 @@ +data "aws_caller_identity" "current" {} + +data "aws_region" "current" {} + +locals { + k8s_service_account_namespace = "kube-system" + k8s_service_account_name = "cluster-autoscaler-aws" +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token + } +} + +resource "helm_release" "cluster-autoscaler" { + depends_on = [ + module.eks + ] + + name = "cluster-autoscaler" + namespace = local.k8s_service_account_namespace + repository = "https://kubernetes.github.io/autoscaler" + chart = "cluster-autoscaler" + version = "9.10.7" + create_namespace = false + + set { + name = "awsRegion" + value = data.aws_region.current.name + } + set { + name = "rbac.serviceAccount.name" + value = local.k8s_service_account_name + } + set { + name = "rbac.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.iam_assumable_role_admin.iam_role_arn + type = "string" + } + set { + name = "autoDiscovery.clusterName" + value = local.name + } + set { + name = "autoDiscovery.enabled" + value = "true" + } + set { + name = "rbac.create" + value = "true" + } +} + module "iam_assumable_role_admin" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "3.6.0" + version = "~> 4.0" create_role = true role_name = "cluster-autoscaler" diff --git a/examples/irsa/locals.tf b/examples/irsa/locals.tf deleted file mode 100644 index a0e5da0c28..0000000000 --- a/examples/irsa/locals.tf +++ /dev/null @@ -1,5 +0,0 @@ -locals { - cluster_name = "test-eks-irsa" - k8s_service_account_namespace = "kube-system" - k8s_service_account_name = "cluster-autoscaler-aws-cluster-autoscaler-chart" -} diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index c9ea505900..ebe2eeffea 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -1,53 +1,37 @@ provider "aws" { - region = "eu-west-1" + region = local.region } -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id +locals { + name = "irsa-${random_string.suffix.result}" + cluster_version = "1.20" + region = "eu-west-1" } -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} +################################################################################ +# EKS Module +################################################################################ -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} +module "eks" { + source = "../.." -data "aws_availability_zones" "available" {} + cluster_name = local.name + cluster_version = local.cluster_version -data "aws_caller_identity" "current" {} + vpc_id = module.vpc.vpc_id + subnets = module.vpc.private_subnets -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "2.64.0" - name = "test-vpc" - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - public_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - enable_dns_hostnames = true + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true - public_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/elb" = "1" - } -} - -module "eks" { - source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.20" - subnets = module.vpc.public_subnets - vpc_id = module.vpc.vpc_id - enable_irsa = true + enable_irsa = true worker_groups = [ { name = "worker-group-1" instance_type = "t3.medium" asg_desired_capacity = 1 + asg_max_size = 4 tags = [ { "key" = "k8s.io/cluster-autoscaler/enabled" @@ -55,11 +39,77 @@ module "eks" { "value" = "true" }, { - "key" = "k8s.io/cluster-autoscaler/${local.cluster_name}" + "key" = "k8s.io/cluster-autoscaler/${local.name}" "propagate_at_launch" = "false" "value" = "owned" } ] } ] + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# Kubernetes provider configuration +################################################################################ + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id } + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} + +################################################################################ +# Supporting Resources +################################################################################ + +data "aws_availability_zones" "available" { +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + diff --git a/examples/irsa/versions.tf b/examples/irsa/versions.tf index 6e29ae8f1b..f546ca0cf0 100644 --- a/examples/irsa/versions.tf +++ b/examples/irsa/versions.tf @@ -5,6 +5,7 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = "~> 1.11" + kubernetes = "~> 2.0" + helm = "~> 2.0" } } diff --git a/examples/_bootstrap/README.md b/examples/launch_templates/README.md similarity index 53% rename from examples/_bootstrap/README.md rename to examples/launch_templates/README.md index 28e34f9c8f..6d1d5e2549 100644 --- a/examples/_bootstrap/README.md +++ b/examples/launch_templates/README.md @@ -1,8 +1,8 @@ -# Various bootstrap resources required for other EKS examples +# Launch templates example -Configuration in this directory creates some resources required in other EKS examples (such as VPC). +This is EKS example using workers launch template with worker groups feature. -The resources created here are free (no NAT gateways here) and they can reside in test AWS account. +See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) for more details. ## Usage @@ -23,7 +23,8 @@ Note that this example may create resources which cost money. Run `terraform des |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.22.0 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | ## Providers @@ -37,6 +38,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| +| [eks](#module\_eks) | ../.. | | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources @@ -45,6 +47,8 @@ Note that this example may create resources which cost money. Run `terraform des |------|------| | [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | ## Inputs @@ -54,7 +58,8 @@ No inputs. | Name | Description | |------|-------------| -| [cluster\_name](#output\_cluster\_name) | Name of EKS Cluster used in tags for subnets | -| [region](#output\_region) | AWS region | -| [vpc](#output\_vpc) | Complete output of VPC module | +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | +| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | +| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index 68fc20599b..476ca13d68 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -1,50 +1,25 @@ provider "aws" { - region = "eu-west-1" -} - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} - -data "aws_availability_zones" "available" { + region = local.region } locals { - cluster_name = "test-eks-lt-${random_string.suffix.result}" -} - -resource "random_string" "suffix" { - length = 8 - special = false + name = "launch_template-${random_string.suffix.result}" + cluster_version = "1.20" + region = "eu-west-1" } -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 2.47" - - name = "test-vpc-lt" - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_dns_hostnames = true -} +################################################################################ +# EKS Module +################################################################################ module "eks" { - source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.20" - subnets = module.vpc.public_subnets - vpc_id = module.vpc.vpc_id + source = "../.." + cluster_name = local.name + cluster_version = local.cluster_version + vpc_id = module.vpc.vpc_id + subnets = module.vpc.private_subnets + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true worker_groups_launch_template = [ { @@ -90,4 +65,70 @@ module "eks" { ] }, ] + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# Kubernetes provider configuration +################################################################################ + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} + +################################################################################ +# Supporting Resources +################################################################################ + +data "aws_availability_zones" "available" { +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } } diff --git a/examples/launch_templates/outputs.tf b/examples/launch_templates/outputs.tf index 359db3a481..b778ec7926 100644 --- a/examples/launch_templates/outputs.tf +++ b/examples/launch_templates/outputs.tf @@ -1,19 +1,19 @@ -output "cluster_endpoint" { - description = "Endpoint for EKS control plane." - value = module.eks.cluster_endpoint -} - -output "cluster_security_group_id" { - description = "Security group ids attached to the cluster control plane." - value = module.eks.cluster_security_group_id -} - -output "kubectl_config" { - description = "kubectl config as generated by the module." - value = module.eks.kubeconfig -} - -output "config_map_aws_auth" { - description = "A kubernetes configuration to authenticate to this EKS cluster." - value = module.eks.config_map_aws_auth -} +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = module.eks.cluster_security_group_id +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = module.eks.kubeconfig +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = module.eks.config_map_aws_auth +} diff --git a/examples/launch_templates/pre_userdata.sh b/examples/launch_templates/pre_userdata.sh index 52dd50f28c..4cbf0d114b 100644 --- a/examples/launch_templates/pre_userdata.sh +++ b/examples/launch_templates/pre_userdata.sh @@ -1 +1 @@ -yum update -y +yum update -y diff --git a/examples/launch_templates/versions.tf b/examples/launch_templates/versions.tf index 6e29ae8f1b..9c1dbfa3e8 100644 --- a/examples/launch_templates/versions.tf +++ b/examples/launch_templates/versions.tf @@ -5,6 +5,7 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = "~> 1.11" + kubernetes = "~> 2.0" } } + diff --git a/examples/launch_templates_with_managed_node_groups/README.md b/examples/launch_templates_with_managed_node_groups/README.md new file mode 100644 index 0000000000..e79ff558f9 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/README.md @@ -0,0 +1,70 @@ +# Launch template with managed groups example + +This is EKS example using workers custom launch template with managed groups feature in two different ways: + +- Using a defined existing launch template created outside module +- Using dlaunch template which will be created by module with user customization + +See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html) for more details. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [local](#requirement\_local) | >= 1.4 | +| [random](#requirement\_random) | >= 2.1 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.22.0 | +| [random](#provider\_random) | >= 2.1 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_service_linked_role.autoscaling](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_service_linked_role) | resource | +| [aws_launch_template.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | +| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | +| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | + diff --git a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf index a2840ebc77..0f0e4ebf31 100644 --- a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf +++ b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf @@ -2,7 +2,7 @@ # template = file("${path.module}/templates/userdata.sh.tpl") # # vars = { -# cluster_name = local.cluster_name +# cluster_name = local.name # endpoint = module.eks.cluster_endpoint # cluster_auth_base64 = module.eks.cluster_certificate_authority_data # @@ -17,6 +17,7 @@ # # Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, # then the default user-data for bootstrapping a cluster is merged in the copy. + resource "aws_launch_template" "default" { name_prefix = "eks-example-" description = "Default Launch-Template" @@ -59,22 +60,21 @@ resource "aws_launch_template" "default" { # data.template_file.launch_template_userdata.rendered, # ) - # Supplying custom tags to EKS instances is another use-case for LaunchTemplates tag_specifications { resource_type = "instance" tags = { - CustomTag = "EKS example" + CustomTag = "Instance custom tag" } } - # Supplying custom tags to EKS instances root volumes is another use-case for LaunchTemplates. (doesnt add tags to dynamically provisioned volumes via PVC tho) + # Supplying custom tags to EKS instances root volumes is another use-case for LaunchTemplates. (doesnt add tags to dynamically provisioned volumes via PVC) tag_specifications { resource_type = "volume" tags = { - CustomTag = "EKS example" + CustomTag = "Volume custom tag" } } @@ -89,7 +89,7 @@ resource "aws_launch_template" "default" { # Tag the LT itself tags = { - CustomTag = "EKS example" + CustomTag = "Launch template custom tag" } lifecycle { diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf index 833b4b9f29..5d70e97615 100644 --- a/examples/launch_templates_with_managed_node_groups/main.tf +++ b/examples/launch_templates_with_managed_node_groups/main.tf @@ -1,7 +1,91 @@ provider "aws" { - region = "eu-west-1" + region = local.region } +locals { + name = "lt_with_mng-${random_string.suffix.result}" + cluster_version = "1.20" + region = "eu-west-1" +} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + + vpc_id = module.vpc.vpc_id + subnets = module.vpc.private_subnets + + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + node_groups = { + # use arleady defined launch template + example1 = { + name_prefix = "example1" + desired_capacity = 1 + max_capacity = 15 + min_capacity = 1 + + launch_template_id = aws_launch_template.default.id + launch_template_version = aws_launch_template.default.default_version + + instance_types = ["t3.small"] + + additional_tags = { + ExtraTag = "example1" + } + } + # create launch template + example2 = { + create_launch_template = true + desired_capacity = 1 + max_capacity = 10 + min_capacity = 1 + + disk_size = 50 + disk_type = "gp3" + disk_throughput = 150 + disk_iops = 3000 + + instance_types = ["t3.large"] + capacity_type = "SPOT" + k8s_labels = { + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + additional_tags = { + ExtraTag = "example2" + } + taints = [ + { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + ] + update_config = { + max_unavailable_percentage = 50 # or set `max_unavailable` + } + } + } + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# Kubernetes provider configuration +################################################################################ + data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } @@ -16,11 +100,11 @@ provider "kubernetes" { token = data.aws_eks_cluster_auth.cluster.token } -data "aws_availability_zones" "available" { -} +################################################################################ +# Supporting Resources +################################################################################ -locals { - cluster_name = "test-eks-lt-${random_string.suffix.result}" +data "aws_availability_zones" "available" { } resource "random_string" "suffix" { @@ -30,43 +114,30 @@ resource "random_string" "suffix" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 2.47" + version = "~> 3.0" - name = "test-vpc" - cidr = "172.16.0.0/16" + name = local.name + cidr = "10.0.0.0/16" azs = data.aws_availability_zones.available.names - private_subnets = ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"] - public_subnets = ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] enable_nat_gateway = true single_nat_gateway = true enable_dns_hostnames = true - private_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" # EKS adds this and TF would want to remove then later + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = "1" } -} - -module "eks" { - source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.20" - subnets = module.vpc.private_subnets - vpc_id = module.vpc.vpc_id - - node_groups = { - example = { - desired_capacity = 1 - max_capacity = 15 - min_capacity = 1 - - launch_template_id = aws_launch_template.default.id - launch_template_version = aws_launch_template.default.default_version - instance_types = var.instance_types + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } - additional_tags = { - CustomTag = "EKS example" - } - } + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" } } diff --git a/examples/launch_templates_with_managed_node_groups/variables.tf b/examples/launch_templates_with_managed_node_groups/variables.tf index 9bd936c6f8..e69de29bb2 100644 --- a/examples/launch_templates_with_managed_node_groups/variables.tf +++ b/examples/launch_templates_with_managed_node_groups/variables.tf @@ -1,6 +0,0 @@ -variable "instance_types" { - description = "Instance types" - # Smallest recommended, where ~1.1Gb of 2Gb memory is available for the Kubernetes pods after ‘warming up’ Docker, Kubelet, and OS - type = list(string) - default = ["t3.small"] -} diff --git a/examples/launch_templates_with_managed_node_groups/versions.tf b/examples/launch_templates_with_managed_node_groups/versions.tf index 6e29ae8f1b..bbcf893252 100644 --- a/examples/launch_templates_with_managed_node_groups/versions.tf +++ b/examples/launch_templates_with_managed_node_groups/versions.tf @@ -5,6 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = "~> 1.11" + kubernetes = "~> 2.0" } } diff --git a/examples/managed_node_groups/README.md b/examples/managed_node_groups/README.md new file mode 100644 index 0000000000..8cc2ecd8ae --- /dev/null +++ b/examples/managed_node_groups/README.md @@ -0,0 +1,73 @@ +# Managed groups example + +This is EKS example using managed groups feature in two different ways: + +- Using SPOT instances in node group +- Using ON_DEMAND instance in node group + +See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for more details. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [local](#requirement\_local) | >= 1.4 | +| [random](#requirement\_random) | >= 2.1 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.22.0 | +| [random](#provider\_random) | >= 2.1 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | + +## Resources + +| Name | Type | +|------|------| +| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [map\_accounts](#input\_map\_accounts) | Additional AWS account numbers to add to the aws-auth configmap. | `list(string)` |
[
"777777777777",
"888888888888"
]
| no | +| [map\_roles](#input\_map\_roles) | Additional IAM roles to add to the aws-auth configmap. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
|
[
{
"groups": [
"system:masters"
],
"rolearn": "arn:aws:iam::66666666666:role/role1",
"username": "role1"
}
]
| no | +| [map\_users](#input\_map\_users) | Additional IAM users to add to the aws-auth configmap. |
list(object({
userarn = string
username = string
groups = list(string)
}))
|
[
{
"groups": [
"system:masters"
],
"userarn": "arn:aws:iam::66666666666:user/user1",
"username": "user1"
},
{
"groups": [
"system:masters"
],
"userarn": "arn:aws:iam::66666666666:user/user2",
"username": "user2"
}
]
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | +| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | +| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | +| [node\_groups](#output\_node\_groups) | Outputs from node groups | + diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf index 99cc1fd730..56a2b05346 100644 --- a/examples/managed_node_groups/main.tf +++ b/examples/managed_node_groups/main.tf @@ -1,70 +1,28 @@ provider "aws" { - region = "eu-west-1" -} - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} - -data "aws_availability_zones" "available" { + region = local.region } locals { - cluster_name = "test-eks-${random_string.suffix.result}" -} - -resource "random_string" "suffix" { - length = 8 - special = false + name = "managed_node_groups-${random_string.suffix.result}" + cluster_version = "1.20" + region = "eu-west-1" } -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 2.47" - - name = "test-vpc" - cidr = "172.16.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"] - public_subnets = ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } -} +################################################################################ +# EKS Module +################################################################################ module "eks" { - source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.20" - subnets = module.vpc.private_subnets + source = "../.." - tags = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } + cluster_name = local.name + cluster_version = local.cluster_version + + vpc_id = module.vpc.vpc_id + subnets = module.vpc.private_subnets - vpc_id = module.vpc.vpc_id + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true node_groups_defaults = { ami_type = "AL2_x86_64" @@ -73,23 +31,16 @@ module "eks" { node_groups = { example = { - create_launch_template = true - desired_capacity = 1 max_capacity = 10 min_capacity = 1 - disk_size = 50 - disk_type = "gp3" - disk_throughput = 150 - disk_iops = 3000 - instance_types = ["t3.large"] capacity_type = "SPOT" k8s_labels = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" + Example = "managed_node_groups" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" } additional_tags = { ExtraTag = "example" @@ -105,24 +56,93 @@ module "eks" { max_unavailable_percentage = 50 # or set `max_unavailable` } } - } - - # Create security group rules to allow communication between pods on workers and pods in managed node groups. - # Set this to true if you have AWS-Managed node groups and Self-Managed worker groups. - # See https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1089 - - # worker_create_cluster_primary_security_group_rules = true + example2 = { + desired_capacity = 1 + max_capacity = 10 + min_capacity = 1 - # worker_groups_launch_template = [ - # { - # name = "worker-group-1" - # instance_type = "t3.small" - # asg_desired_capacity = 2 - # public_ip = true - # } - # ] + instance_types = ["t3.medium"] + k8s_labels = { + Example = "managed_node_groups" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + additional_tags = { + ExtraTag = "example2" + } + update_config = { + max_unavailable_percentage = 50 # or set `max_unavailable` + } + } + } map_roles = var.map_roles map_users = var.map_users map_accounts = var.map_accounts + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# Kubernetes provider configuration +################################################################################ + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} + +################################################################################ +# Supporting Resources +################################################################################ + +data "aws_availability_zones" "available" { +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } } diff --git a/examples/managed_node_groups/versions.tf b/examples/managed_node_groups/versions.tf index 47f7c05e43..bbcf893252 100644 --- a/examples/managed_node_groups/versions.tf +++ b/examples/managed_node_groups/versions.tf @@ -2,9 +2,9 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.56.0" + aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = "~> 1.11" + kubernetes = "~> 2.0" } } diff --git a/examples/secrets_encryption/README.md b/examples/secrets_encryption/README.md new file mode 100644 index 0000000000..c3ee731f2a --- /dev/null +++ b/examples/secrets_encryption/README.md @@ -0,0 +1,66 @@ +# Managed groups example + +This is EKS using [secrets encryption](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) feature. + +See [the official blog](https://aws.amazon.com/blogs/containers/using-eks-encryption-provider-support-for-defense-in-depth/) for more details. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.22.0 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [local](#requirement\_local) | >= 1.4 | +| [random](#requirement\_random) | >= 2.1 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.22.0 | +| [random](#provider\_random) | >= 2.1 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | +| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | +| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | + diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index 76fa2b2392..49d9a7b029 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -1,7 +1,57 @@ provider "aws" { - region = "eu-west-1" + region = local.region } +locals { + name = "secrets_encryption-${random_string.suffix.result}" + cluster_version = "1.20" + region = "eu-west-1" +} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + + vpc_id = module.vpc.vpc_id + subnets = module.vpc.private_subnets + + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + + cluster_encryption_config = [ + { + provider_key_arn = aws_kms_key.eks.arn + resources = ["secrets"] + } + ] + + worker_groups = [ + { + name = "worker-group-1" + instance_type = "t3.small" + additional_userdata = "echo foo bar" + asg_desired_capacity = 2 + }, + ] + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# Kubernetes provider configuration +################################################################################ + data "aws_eks_cluster" "cluster" { name = module.eks.cluster_id } @@ -16,11 +66,28 @@ provider "kubernetes" { token = data.aws_eks_cluster_auth.cluster.token } -data "aws_availability_zones" "available" { +################################################################################ +# KMS for encrypting secrets +################################################################################ + +resource "aws_kms_key" "eks" { + description = "EKS Secret Encryption Key" + deletion_window_in_days = 7 + enable_key_rotation = true + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } } -locals { - cluster_name = "test-eks-${random_string.suffix.result}" + +################################################################################ +# Supporting Resources +################################################################################ + +data "aws_availability_zones" "available" { } resource "random_string" "suffix" { @@ -28,15 +95,11 @@ resource "random_string" "suffix" { special = false } -resource "aws_kms_key" "eks" { - description = "EKS Secret Encryption Key" -} - module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 2.47" + version = "~> 3.0" - name = "test-vpc" + name = local.name cidr = "10.0.0.0/16" azs = data.aws_availability_zones.available.names private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] @@ -46,47 +109,18 @@ module "vpc" { enable_dns_hostnames = true public_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/elb" = "1" + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = "1" } private_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" } -} - -module "eks" { - source = "../.." - cluster_name = local.cluster_name - cluster_version = "1.20" - subnets = module.vpc.private_subnets - - cluster_encryption_config = [ - { - provider_key_arn = aws_kms_key.eks.arn - resources = ["secrets"] - } - ] tags = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" } - - vpc_id = module.vpc.vpc_id - - worker_groups = [ - { - name = "worker-group-1" - instance_type = "t3.small" - additional_userdata = "echo foo bar" - asg_desired_capacity = 2 - }, - ] - - map_roles = var.map_roles - map_users = var.map_users - map_accounts = var.map_accounts } diff --git a/examples/secrets_encryption/variables.tf b/examples/secrets_encryption/variables.tf index 57853d8b4d..e69de29bb2 100644 --- a/examples/secrets_encryption/variables.tf +++ b/examples/secrets_encryption/variables.tf @@ -1,48 +0,0 @@ -variable "map_accounts" { - description = "Additional AWS account numbers to add to the aws-auth configmap." - type = list(string) - - default = [ - "777777777777", - "888888888888", - ] -} - -variable "map_roles" { - description = "Additional IAM roles to add to the aws-auth configmap." - type = list(object({ - rolearn = string - username = string - groups = list(string) - })) - - default = [ - { - rolearn = "arn:aws:iam::66666666666:role/role1" - username = "role1" - groups = ["system:masters"] - }, - ] -} - -variable "map_users" { - description = "Additional IAM users to add to the aws-auth configmap." - type = list(object({ - userarn = string - username = string - groups = list(string) - })) - - default = [ - { - userarn = "arn:aws:iam::66666666666:user/user1" - username = "user1" - groups = ["system:masters"] - }, - { - userarn = "arn:aws:iam::66666666666:user/user2" - username = "user2" - groups = ["system:masters"] - }, - ] -} diff --git a/examples/secrets_encryption/versions.tf b/examples/secrets_encryption/versions.tf index 120d873e2f..bbcf893252 100644 --- a/examples/secrets_encryption/versions.tf +++ b/examples/secrets_encryption/versions.tf @@ -5,6 +5,6 @@ terraform { aws = ">= 3.22.0" local = ">= 1.4" random = ">= 2.1" - kubernetes = ">= 1.11" + kubernetes = "~> 2.0" } } diff --git a/variables.tf b/variables.tf index a5d9adefe3..a830e20a62 100644 --- a/variables.tf +++ b/variables.tf @@ -71,13 +71,13 @@ variable "aws_auth_additional_labels" { } variable "map_accounts" { - description = "Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format." + description = "Additional AWS account numbers to add to the aws-auth configmap." type = list(string) default = [] } variable "map_roles" { - description = "Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format." + description = "Additional IAM roles to add to the aws-auth configmap." type = list(object({ rolearn = string username = string @@ -87,7 +87,7 @@ variable "map_roles" { } variable "map_users" { - description = "Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format." + description = "Additional IAM users to add to the aws-auth configmap." type = list(object({ userarn = string username = string From 306ad727f3e2553ab576c5358e00af99553f680a Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Tue, 12 Oct 2021 15:21:42 +0200 Subject: [PATCH 405/962] Updated CHANGELOG --- CHANGELOG.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edfa4df99e..de3b9ea7fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,28 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.21.0] - 2021-10-12 +FEATURES: +- Fix custom AMI bootstrap ([#1580](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1580)) +- Enable throughput & iops configs for managed node_groups ([#1584](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1584)) +- Allow snapshot_id to be specified for additional_ebs_volumes ([#1431](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1431)) +- Allow interface_type to be specified in worker_groups_launch_template ([#1439](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1439)) + +BUG FIXES: +- Rebuild examples ([#1625](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1625)) +- Bug with data source in managed groups submodule ([#1633](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1633)) +- Fixed launch_templates_with_managed_node_group example ([#1599](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1599)) + +DOCS: +- Update iam-permissions.md ([#1613](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1613)) +- Updated iam-permissions.md ([#1612](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1612)) +- Updated faq about desired count of instances in node and worker groups ([#1604](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1604)) +- Update faq about endpoints ([#1603](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1603)) +- Fix broken URL in README ([#1602](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1602)) +- Remove `asg_recreate_on_change` in faq ([#1596](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1596)) + + ## [v17.20.0] - 2021-09-17 FEATURES: @@ -522,7 +544,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.20.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.21.0...HEAD +[v17.21.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.20.0...v17.21.0 [v17.20.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.19.0...v17.20.0 [v17.19.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.18.0...v17.19.0 [v17.18.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.17.0...v17.18.0 From 7c335546d6f755ca2e49488476e1b56bde395251 Mon Sep 17 00:00:00 2001 From: Steve Hipwell Date: Thu, 14 Oct 2021 08:47:27 +0100 Subject: [PATCH 406/962] fix: MNG cluster datasource errors (#1639) --- modules/node_groups/README.md | 3 ++- modules/node_groups/launch_template.tf | 4 ++-- modules/node_groups/locals.tf | 6 ------ modules/node_groups/templates/userdata.sh.tpl | 2 +- modules/node_groups/variables.tf | 12 ++++++++++++ node_groups.tf | 5 ++++- 6 files changed, 21 insertions(+), 11 deletions(-) diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index b7c20cad04..1474da02dc 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -84,13 +84,14 @@ No modules. |------|------| | [aws_eks_node_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_launch_template.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | -| [aws_eks_cluster.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | | [cloudinit_config.workers_userdata](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of parent cluster | `string` | `""` | no | +| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of parent cluster | `string` | `""` | no | | [cluster\_name](#input\_cluster\_name) | Name of parent cluster | `string` | `""` | no | | [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | [default\_iam\_role\_arn](#input\_default\_iam\_role\_arn) | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | `""` | no | diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 5c2243d4c5..fcf65b7c9a 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -14,8 +14,8 @@ data "cloudinit_config" "workers_userdata" { ami_id = lookup(each.value, "ami_id", "") ami_is_eks_optimized = each.value["ami_is_eks_optimized"] cluster_name = var.cluster_name - cluster_endpoint = data.aws_eks_cluster.default[0].endpoint - cluster_ca = data.aws_eks_cluster.default[0].certificate_authority[0].data + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 capacity_type = lookup(each.value, "capacity_type", "ON_DEMAND") append_labels = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : "" } diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 490a42d306..0b211721e3 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -1,9 +1,3 @@ -data "aws_eks_cluster" "default" { - count = var.create_eks ? 1 : 0 - - name = var.cluster_name -} - locals { # Merge defaults and per-group values to make code cleaner node_groups_expanded = { for k, v in var.node_groups : k => merge( diff --git a/modules/node_groups/templates/userdata.sh.tpl b/modules/node_groups/templates/userdata.sh.tpl index aab0c5ad79..d9a590bcc8 100644 --- a/modules/node_groups/templates/userdata.sh.tpl +++ b/modules/node_groups/templates/userdata.sh.tpl @@ -7,7 +7,7 @@ sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" ${kubelet_extra_args}"' /e # Set variables for custom AMI API_SERVER_URL=${cluster_endpoint} -B64_CLUSTER_CA=${cluster_ca} +B64_CLUSTER_CA=${cluster_auth_base64} KUBELET_EXTRA_ARGS='--node-labels=eks.amazonaws.com/nodegroup-image=${ami_id},eks.amazonaws.com/capacityType=${capacity_type}${append_labels} ${kubelet_extra_args}' %{endif ~} diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf index d106bb7a65..e9e259f13f 100644 --- a/modules/node_groups/variables.tf +++ b/modules/node_groups/variables.tf @@ -10,6 +10,18 @@ variable "cluster_name" { default = "" } +variable "cluster_endpoint" { + description = "Endpoint of parent cluster" + type = string + default = "" +} + +variable "cluster_auth_base64" { + description = "Base64 encoded CA of parent cluster" + type = string + default = "" +} + variable "default_iam_role_arn" { description = "ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults`" type = string diff --git a/node_groups.tf b/node_groups.tf index c40f635208..206bf8ed48 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -3,7 +3,10 @@ module "node_groups" { create_eks = var.create_eks - cluster_name = local.cluster_name + cluster_name = local.cluster_name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + default_iam_role_arn = coalescelist(aws_iam_role.workers[*].arn, [""])[0] ebs_optimized_not_supported = local.ebs_optimized_not_supported workers_group_defaults = local.workers_group_defaults From 6b3a8e62379e4438df73ad18921805436d917482 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Thu, 14 Oct 2021 09:47:56 +0200 Subject: [PATCH 407/962] Updated CHANGELOG --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de3b9ea7fe..da18dd83bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.22.0] - 2021-10-14 +BUG FIXES: +- MNG cluster datasource errors ([#1639](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1639)) + + ## [v17.21.0] - 2021-10-12 FEATURES: @@ -544,7 +550,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.21.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.22.0...HEAD +[v17.22.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.21.0...v17.22.0 [v17.21.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.20.0...v17.21.0 [v17.20.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.19.0...v17.20.0 [v17.19.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.18.0...v17.19.0 From bc0988cf803b59d207014bd6e8a4963eb8b4442c Mon Sep 17 00:00:00 2001 From: Tamas Millian Date: Mon, 18 Oct 2021 13:26:38 +0100 Subject: [PATCH 408/962] fix: Fixed variable reference for snapshot_id (#1634) --- workers.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers.tf b/workers.tf index 8dc2754e56..610c41709e 100644 --- a/workers.tf +++ b/workers.tf @@ -345,7 +345,7 @@ resource "aws_launch_configuration" "workers" { local.workers_group_defaults["root_encrypted"], ) snapshot_id = lookup( - block_device_mappings.value, + ebs_block_device.value, "snapshot_id", local.workers_group_defaults["snapshot_id"], ) From 949bebe5bcd9b10f3d2295ee8ba407ca95ee02b5 Mon Sep 17 00:00:00 2001 From: Steve Hipwell Date: Tue, 2 Nov 2021 09:56:44 +0000 Subject: [PATCH 409/962] feat: Improve managed node group bootstrap revisited (#1577) --- modules/node_groups/README.md | 3 ++- modules/node_groups/launch_template.tf | 9 +++++---- modules/node_groups/locals.tf | 1 + modules/node_groups/templates/userdata.sh.tpl | 18 ++++++++++++++++-- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 1474da02dc..98936d6a9b 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -37,7 +37,8 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | instance\_types | Node group's instance type(s). Multiple types can be specified when `capacity_type="SPOT"`. | list | `[var.workers_group_defaults[instance_type]]` | | k8s\_labels | Kubernetes labels | map(string) | No labels applied | | key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` | -| kubelet_extra_args | This string is passed directly to kubelet if set. Useful for adding labels or taints. Require `create_launch_template` to be `true`| string | "" | +| bootstrap_env | Provide environment variables to customise [bootstrap.sh](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh). Require `create_launch_template` to be `true` | map(string) | `{}` | +| kubelet_extra_args | Extra arguments for kubelet, this is automatically merged with `labels`. Require `create_launch_template` to be `true` | string | "" | | launch_template_id | The id of a aws_launch_template to use | string | No LT used | | launch\_template_version | The version of the LT to use | string | none | | max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` | diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index fcf65b7c9a..6abe358d5a 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -9,13 +9,14 @@ data "cloudinit_config" "workers_userdata" { content_type = "text/x-shellscript" content = templatefile("${path.module}/templates/userdata.sh.tpl", { - kubelet_extra_args = each.value["kubelet_extra_args"] - pre_userdata = each.value["pre_userdata"] - ami_id = lookup(each.value, "ami_id", "") - ami_is_eks_optimized = each.value["ami_is_eks_optimized"] cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 + ami_id = lookup(each.value, "ami_id", "") + ami_is_eks_optimized = each.value["ami_is_eks_optimized"] + bootstrap_env = each.value["bootstrap_env"] + kubelet_extra_args = each.value["kubelet_extra_args"] + pre_userdata = each.value["pre_userdata"] capacity_type = lookup(each.value, "capacity_type", "ON_DEMAND") append_labels = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : "" } diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 0b211721e3..0a6c7cbffb 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -13,6 +13,7 @@ locals { min_capacity = var.workers_group_defaults["asg_min_size"] subnets = var.workers_group_defaults["subnets"] create_launch_template = false + bootstrap_env = {} kubelet_extra_args = var.workers_group_defaults["kubelet_extra_args"] disk_size = var.workers_group_defaults["root_volume_size"] disk_type = var.workers_group_defaults["root_volume_type"] diff --git a/modules/node_groups/templates/userdata.sh.tpl b/modules/node_groups/templates/userdata.sh.tpl index d9a590bcc8..321c17b427 100644 --- a/modules/node_groups/templates/userdata.sh.tpl +++ b/modules/node_groups/templates/userdata.sh.tpl @@ -1,13 +1,27 @@ #!/bin/bash -e %{ if length(ami_id) == 0 ~} -# Set variables directly into bootstrap.sh for default AMI -sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" ${kubelet_extra_args}"' /etc/eks/bootstrap.sh +# Set bootstrap env +printf '#!/bin/bash +%{ for k, v in bootstrap_env ~} +export ${k}="${v}" +%{ endfor ~} +export ADDITIONAL_KUBELET_EXTRA_ARGS="${kubelet_extra_args}" +' > /etc/profile.d/eks-bootstrap-env.sh + +# Source extra environment variables in bootstrap script +sed -i '/^set -o errexit/a\\nsource /etc/profile.d/eks-bootstrap-env.sh' /etc/eks/bootstrap.sh + +# Merge ADDITIONAL_KUBELET_EXTRA_ARGS into KUBELET_EXTRA_ARGS +sed -i 's/^KUBELET_EXTRA_ARGS="$${KUBELET_EXTRA_ARGS:-}/KUBELET_EXTRA_ARGS="$${KUBELET_EXTRA_ARGS:-} $${ADDITIONAL_KUBELET_EXTRA_ARGS}/' /etc/eks/bootstrap.sh %{else ~} # Set variables for custom AMI API_SERVER_URL=${cluster_endpoint} B64_CLUSTER_CA=${cluster_auth_base64} +%{ for k, v in bootstrap_env ~} +${k}="${v}" +%{ endfor ~} KUBELET_EXTRA_ARGS='--node-labels=eks.amazonaws.com/nodegroup-image=${ami_id},eks.amazonaws.com/capacityType=${capacity_type}${append_labels} ${kubelet_extra_args}' %{endif ~} From ed048f3c1af31f75dddb76430a43f60e6c4dfdf9 Mon Sep 17 00:00:00 2001 From: "Vilvaramadurai Samidurai (Vilva)" Date: Tue, 2 Nov 2021 10:19:40 +0000 Subject: [PATCH 410/962] feat: Added support for client.authentication.k8s.io/v1beta1 (#1550) --- README.md | 1 + locals.tf | 15 ++++++++------- templates/kubeconfig.tpl | 2 +- variables.tf | 5 +++++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ab4fe8b1ec..d4dd2afff4 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [fargate\_profiles](#input\_fargate\_profiles) | Fargate profiles to create. See `fargate_profile` keys section in fargate submodule's README.md for more details | `any` | `{}` | no | | [fargate\_subnets](#input\_fargate\_subnets) | A list of subnets to place fargate workers within (if different from subnets). | `list(string)` | `[]` | no | | [iam\_path](#input\_iam\_path) | If provided, all IAM roles will be created on this path. | `string` | `"/"` | no | +| [kubeconfig\_api\_version](#input\_kubeconfig\_api\_version) | KubeConfig API version. Defaults to client.authentication.k8s.io/v1alpha1 | `string` | `"client.authentication.k8s.io/v1alpha1"` | no | | [kubeconfig\_aws\_authenticator\_additional\_args](#input\_kubeconfig\_aws\_authenticator\_additional\_args) | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | `list(string)` | `[]` | no | | [kubeconfig\_aws\_authenticator\_command](#input\_kubeconfig\_aws\_authenticator\_command) | Command to use to fetch AWS EKS credentials. | `string` | `"aws-iam-authenticator"` | no | | [kubeconfig\_aws\_authenticator\_command\_args](#input\_kubeconfig\_aws\_authenticator\_command\_args) | Default arguments passed to the authenticator command. Defaults to [token -i $cluster\_name]. | `list(string)` | `[]` | no | diff --git a/locals.tf b/locals.tf index 09dd589214..8e1b54e49c 100644 --- a/locals.tf +++ b/locals.tf @@ -163,13 +163,14 @@ locals { ] kubeconfig = var.create_eks ? templatefile("${path.module}/templates/kubeconfig.tpl", { - kubeconfig_name = coalesce(var.kubeconfig_name, "eks_${var.cluster_name}") - endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 - aws_authenticator_command = var.kubeconfig_aws_authenticator_command - aws_authenticator_command_args = coalescelist(var.kubeconfig_aws_authenticator_command_args, ["token", "-i", local.cluster_name]) - aws_authenticator_additional_args = var.kubeconfig_aws_authenticator_additional_args - aws_authenticator_env_variables = var.kubeconfig_aws_authenticator_env_variables + kubeconfig_name = coalesce(var.kubeconfig_name, "eks_${var.cluster_name}") + endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + aws_authenticator_kubeconfig_apiversion = var.kubeconfig_api_version + aws_authenticator_command = var.kubeconfig_aws_authenticator_command + aws_authenticator_command_args = coalescelist(var.kubeconfig_aws_authenticator_command_args, ["token", "-i", local.cluster_name]) + aws_authenticator_additional_args = var.kubeconfig_aws_authenticator_additional_args + aws_authenticator_env_variables = var.kubeconfig_aws_authenticator_env_variables }) : "" launch_configuration_userdata_rendered = [ diff --git a/templates/kubeconfig.tpl b/templates/kubeconfig.tpl index a99a0dfa8f..5004243bec 100644 --- a/templates/kubeconfig.tpl +++ b/templates/kubeconfig.tpl @@ -20,7 +20,7 @@ users: - name: ${kubeconfig_name} user: exec: - apiVersion: client.authentication.k8s.io/v1alpha1 + apiVersion: ${aws_authenticator_kubeconfig_apiversion} command: ${aws_authenticator_command} args: %{~ for i in aws_authenticator_command_args } diff --git a/variables.tf b/variables.tf index a830e20a62..2b2763bb5f 100644 --- a/variables.tf +++ b/variables.tf @@ -191,7 +191,12 @@ variable "workers_additional_policies" { type = list(string) default = [] } +variable "kubeconfig_api_version" { + description = "KubeConfig API version. Defaults to client.authentication.k8s.io/v1alpha1" + type = string + default = "client.authentication.k8s.io/v1alpha1" +} variable "kubeconfig_aws_authenticator_command" { description = "Command to use to fetch AWS EKS credentials." type = string From 967403be73af047c642105ec0935fddb3bffab9f Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Tue, 2 Nov 2021 11:20:40 +0100 Subject: [PATCH 411/962] Updated CHANGELOG --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da18dd83bb..fe6345f9b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,16 @@ project adheres to [Semantic Versioning](http://semver.org/). + +## [v17.23.0] - 2021-11-02 +FEATURES: +- Added support for client.authentication.k8s.io/v1beta1 ([#1550](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1550)) +- Improve managed node group bootstrap revisited ([#1577](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1577)) + +BUG FIXES: +- Fixed variable reference for snapshot_id ([#1634](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1634)) + + ## [v17.22.0] - 2021-10-14 BUG FIXES: @@ -550,7 +560,8 @@ CI: - Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) -[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.22.0...HEAD +[Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.23.0...HEAD +[v17.23.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.22.0...v17.23.0 [v17.22.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.21.0...v17.22.0 [v17.21.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.20.0...v17.21.0 [v17.20.0]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.19.0...v17.20.0 From f16cd370606a33e557b6ad1e5e7fa0780d60106e Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Tue, 2 Nov 2021 14:03:11 +0100 Subject: [PATCH 412/962] chore: Migrate from the stalebot app to GitHub Action (#1665) --- .github/stale.yml | 35 ---------------------------- .github/workflows/stale-actions.yaml | 32 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 35 deletions(-) delete mode 100644 .github/stale.yml create mode 100644 .github/workflows/stale-actions.yaml diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index f97e68985d..0000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -# Number of days of inactivity before an issue becomes stale -daysUntilStale: 30 - -# Number of days of inactivity before an stale issue is closed -daysUntilClose: 7 - -# Label to use when marking an issue as stale -staleLabel: stale - -issues: - # Comment to post when marking an issue as stale. - markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. - # Comment to post when closing a stale issue. - closeComment: > - This issue has been automatically closed because it has not had recent - activity since being marked as stale. -pulls: - # Comment to post when marking a PR as stale. - markComment: > - This PR has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. - - To track this PR (even if closed), please open a corresponding issue if one - does not already exist. - # Comment to post when closing a stale PR. - closeComment: > - This PR has been automatically closed because it has not had recent - activity since being marked as stale. - - Please reopen when work resumes. diff --git a/.github/workflows/stale-actions.yaml b/.github/workflows/stale-actions.yaml new file mode 100644 index 0000000000..93c90dfcd1 --- /dev/null +++ b/.github/workflows/stale-actions.yaml @@ -0,0 +1,32 @@ +name: "Mark or close stale issues and PRs" +on: + schedule: + - cron: "0 0 * * *" + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v4 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + # Staling issues and PR's + days-before-stale: 30 + stale-issue-label: stale + stale-pr-label: stale + stale-issue-message: | + This issue has been automatically marked as stale because it has been open 30 days + with no activity. Remove stale label or comment or this issue will be closed in 10 days + stale-pr-message: | + This PR has been automatically marked as stale because it has been open 30 days + with no activity. Remove stale label or comment or this PR will be closed in 10 days + # Not stale if have this labels or part of milestone + exempt-issue-labels: bug,wip,on-hold + exempt-pr-labels: bug,wip,on-hold + exempt-all-milestones: true + # Close issue operations + # Label will be automatically removed if the issues are no longer closed nor locked. + days-before-close: 10 + delete-branch: true + close-issue-message: This issue was automatically closed because of stale in 10 days + close-pr-message: This PR was automatically closed because of stale in 10 days \ No newline at end of file From 8334d0d40d41886feafc3efb156fb789a6a249f3 Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Tue, 2 Nov 2021 14:57:49 +0100 Subject: [PATCH 413/962] chore: Fixed CI (replaced git protocol with https) (#1666) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4fd440d9b6..d26bea60b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.50.0 + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.55.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -20,7 +20,7 @@ repos: - '--args=--only=terraform_required_providers' - '--args=--only=terraform_standard_module_structure' - '--args=--only=terraform_workspace_remote' - - repo: git://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: - id: check-merge-conflict From 6959b9bae32309357bc97a85a1f09c7b590c8a6d Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Tue, 2 Nov 2021 15:34:12 +0100 Subject: [PATCH 414/962] fix: Added Deny for CreateLogGroup action in EKS cluster role (#1594) --- README.md | 3 +++ main.tf | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/README.md b/README.md index d4dd2afff4..56f61a41df 100644 --- a/README.md +++ b/README.md @@ -149,12 +149,14 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [aws_iam_instance_profile.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_instance_profile.workers_launch_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_openid_connect_provider.oidc_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource | +| [aws_iam_policy.cluster_deny_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.cluster_elb_sl_role_creation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cluster_AmazonEKSVPCResourceControllerPolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cluster_deny_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cluster_elb_sl_role_creation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | @@ -183,6 +185,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [aws_iam_instance_profile.custom_worker_group_iam_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_instance_profile) | data source | | [aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_instance_profile) | data source | | [aws_iam_policy_document.cluster_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cluster_deny_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cluster_elb_sl_role_creation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.workers_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_role.custom_cluster_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | diff --git a/main.tf b/main.tf index 328f279dc7..d1860f0696 100644 --- a/main.tf +++ b/main.tf @@ -195,3 +195,38 @@ resource "aws_iam_role_policy_attachment" "cluster_elb_sl_role_creation" { policy_arn = aws_iam_policy.cluster_elb_sl_role_creation[0].arn role = local.cluster_iam_role_name } + +/* + Adding a policy to cluster IAM role that deny permissions to logs:CreateLogGroup + it is not needed since we create the log group ourselve in this module, and it is causing trouble during cleanup/deletion +*/ + +data "aws_iam_policy_document" "cluster_deny_log_group" { + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + + statement { + effect = "Deny" + actions = [ + "logs:CreateLogGroup" + ] + resources = ["*"] + } +} + +resource "aws_iam_policy" "cluster_deny_log_group" { + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + + name_prefix = "${var.cluster_name}-deny-log-group" + description = "Deny CreateLogGroup" + policy = data.aws_iam_policy_document.cluster_deny_log_group[0].json + path = var.iam_path + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "cluster_deny_log_group" { + count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + + policy_arn = aws_iam_policy.cluster_deny_log_group[0].arn + role = local.cluster_iam_role_name +} From b7539dc220f6b5fe199d67569b6f3619ec00fdf0 Mon Sep 17 00:00:00 2001 From: Ivan Dechovski Date: Wed, 3 Nov 2021 10:44:26 +0100 Subject: [PATCH 415/962] feat: Add ability to define custom timeout for fargate profiles (#1614) --- examples/complete/main.tf | 5 +++++ examples/fargate/main.tf | 9 +++++++++ modules/fargate/README.md | 1 + modules/fargate/main.tf | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index ba762e8e03..5cfcab668b 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -114,6 +114,11 @@ module "eks" { tags = { Owner = "test" } + + timeouts = { + create = "20m" + delete = "20m" + } } } diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf index 869ed7e991..09e2e57e2d 100644 --- a/examples/fargate/main.tf +++ b/examples/fargate/main.tf @@ -68,6 +68,11 @@ module "eks" { tags = { Owner = "default" } + + timeouts = { + create = "20m" + delete = "20m" + } } secondary = { @@ -154,6 +159,10 @@ module "fargate_profile_existing_cluster" { Owner = "profile2" submodule = "true" } + + timeouts = { + delete = "20m" + } } } diff --git a/modules/fargate/README.md b/modules/fargate/README.md index c4dfaaf6fd..466b2a051a 100644 --- a/modules/fargate/README.md +++ b/modules/fargate/README.md @@ -15,6 +15,7 @@ See example code in `examples/fargate`. | name | Fargate profile name | `string` | Auto generated in the following format `[cluster_name]-fargate-[fargate_profile_map_key]`| no | | selectors | A list of Kubernetes selectors. See examples/fargate/main.tf for example format. |
list(map({
namespace = string
labels = map(string)
}))
| `[]` | no | | subnets | List of subnet IDs. Will replace the root module subnets. | `list(string)` | `var.subnets` | no | +| timeouts | A map of timeouts for create/delete operations. | `map(string)` | Provider default behavior | no | | tags | Key-value map of resource tags. Will be merged with root module tags. | `map(string)` | `var.tags` | no | diff --git a/modules/fargate/main.tf b/modules/fargate/main.tf index 74e3c9b2e2..a4e4b0d80f 100644 --- a/modules/fargate/main.tf +++ b/modules/fargate/main.tf @@ -63,5 +63,10 @@ resource "aws_eks_fargate_profile" "this" { } } + timeouts { + create = try(each.value["timeouts"].create, null) + delete = try(each.value["timeouts"].delete, null) + } + tags = merge(var.tags, lookup(each.value, "tags", {})) } From c69c39487bbdff8e289678e8f24da176607ac2af Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Wed, 3 Nov 2021 11:37:27 +0100 Subject: [PATCH 416/962] chore: Extend example launch_templates_with_managed_node_groups (#1667) --- examples/launch_templates_with_managed_node_groups/main.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf index 5d70e97615..4392c606fa 100644 --- a/examples/launch_templates_with_managed_node_groups/main.tf +++ b/examples/launch_templates_with_managed_node_groups/main.tf @@ -55,6 +55,12 @@ module "eks" { instance_types = ["t3.large"] capacity_type = "SPOT" + + bootstrap_env = { + CONTAINER_RUNTIME = "containerd" + USE_MAX_PODS = false + } + kubelet_extra_args = "--max-pods=110" k8s_labels = { GithubRepo = "terraform-aws-eks" GithubOrg = "terraform-aws-modules" From 4e54eaac16f8e1df0c71abb56eca71633a8a0b41 Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Fri, 5 Nov 2021 11:12:26 +0100 Subject: [PATCH 417/962] chore: update `cluster_version` description (#1671) --- README.md | 2 +- variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 56f61a41df..5980b48201 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of tags to add to just the eks resource. | `map(string)` | `{}` | no | | [cluster\_update\_timeout](#input\_cluster\_update\_timeout) | Timeout value when updating the EKS cluster. | `string` | `"60m"` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes version to use for the EKS cluster. | `string` | `null` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes minor version to use for the EKS cluster (for example 1.21). | `string` | `null` | no | | [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | [create\_fargate\_pod\_execution\_role](#input\_create\_fargate\_pod\_execution\_role) | Controls if the EKS Fargate pod execution IAM role should be created. | `bool` | `true` | no | | [default\_platform](#input\_default\_platform) | Default platform name. Valid options are `linux` and `windows`. | `string` | `"linux"` | no | diff --git a/variables.tf b/variables.tf index 2b2763bb5f..575c76a33c 100644 --- a/variables.tf +++ b/variables.tf @@ -29,7 +29,7 @@ variable "cluster_security_group_id" { } variable "cluster_version" { - description = "Kubernetes version to use for the EKS cluster." + description = "Kubernetes minor version to use for the EKS cluster (for example 1.21)." type = string default = null } From fb3a7ce450d42ebab1395720ed5f1ca49d6e14bb Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Sat, 6 Nov 2021 20:14:05 +0100 Subject: [PATCH 418/962] docs: update faq about dedicated subnets for control plane (#1674) --- docs/faq.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index 9fc55cc28f..d1a5d0f4bc 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -199,3 +199,37 @@ Both can be used together in the same cluster. This happen because Core DNS can be scheduled on Self-Managed worker groups and by default, the terraform module doesn't create security group rules to ensure communication between pods schedulled on Self-Managed worker group and AWS-Managed node groups. You can set `var.worker_create_cluster_primary_security_group_rules` to `true` to create required rules. + +## Dedicated control plane subnets + +[AWS recommends](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html) to create dedicated subnets for EKS created network interfaces (control plane). The module fully supports this approach. To set up this, you must configure the module by adding additional `subnets` into workers default specification `workers_group_defaults` map or directly `subnets` definition in worker definition. + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + + cluster_version = "1.21" + cluster_name = "my-cluster" + vpc_id = "vpc-1234556abcdef" + subnets = ["subnet-abcde123", "subnet-abcde456", "subnet-abcde789"] + + workers_group_defaults = { + subnets = ["subnet-xyz123", "subnet-xyz456", "subnet-xyz789"] + } + + worker_groups = [ + { + instance_type = "m4.large" + asg_max_size = 5 + }, + { + name = "worker-group-2" + subnets = ["subnet-qwer123"] + instance_type = "t3.medium" + asg_desired_capacity = 1 + public_ip = true + ebs_optimized = true + } + ] +} +``` \ No newline at end of file From 56e93d77de58f311f1d1d7051f40bf77e7b03524 Mon Sep 17 00:00:00 2001 From: Dawid Rogaczewski <20227477+daroga0002@users.noreply.github.com> Date: Sat, 6 Nov 2021 20:19:03 +0100 Subject: [PATCH 419/962] feat: Removed ng_depends_on variable and related hack (#1672) --- modules/node_groups/README.md | 1 - modules/node_groups/main.tf | 1 - modules/node_groups/variables.tf | 8 -------- node_groups.tf | 8 +------- 4 files changed, 1 insertion(+), 17 deletions(-) diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 98936d6a9b..e0bf4f3180 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -97,7 +97,6 @@ No modules. | [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | [default\_iam\_role\_arn](#input\_default\_iam\_role\_arn) | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | `""` | no | | [ebs\_optimized\_not\_supported](#input\_ebs\_optimized\_not\_supported) | List of instance types that do not support EBS optimization | `list(string)` | `[]` | no | -| [ng\_depends\_on](#input\_ng\_depends\_on) | List of references to other resources this submodule depends on | `any` | `null` | no | | [node\_groups](#input\_node\_groups) | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | | [node\_groups\_defaults](#input\_node\_groups\_defaults) | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | diff --git a/modules/node_groups/main.tf b/modules/node_groups/main.tf index 7ceafb53fc..75e6209730 100644 --- a/modules/node_groups/main.tf +++ b/modules/node_groups/main.tf @@ -102,5 +102,4 @@ resource "aws_eks_node_group" "workers" { ignore_changes = [scaling_config[0].desired_size] } - depends_on = [var.ng_depends_on] } diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf index e9e259f13f..1aa8cfe26d 100644 --- a/modules/node_groups/variables.tf +++ b/modules/node_groups/variables.tf @@ -64,14 +64,6 @@ variable "node_groups" { default = {} } -# Hack for a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 -# Will be removed in Terraform 0.13 with the support of module's `depends_on` https://github.com/hashicorp/terraform/issues/10462 -variable "ng_depends_on" { - description = "List of references to other resources this submodule depends on" - type = any - default = null -} - variable "ebs_optimized_not_supported" { description = "List of instance types that do not support EBS optimization" type = list(string) diff --git a/node_groups.tf b/node_groups.tf index 206bf8ed48..531a3df480 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -18,14 +18,8 @@ module "node_groups" { tags = var.tags - # Hack to ensure ordering of resource creation. - # This is a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 - # Do not create node_groups before other resources are ready and removes race conditions - # Ensure these resources are created before "unlocking" the data source. - # Will be removed in Terraform 0.13 - ng_depends_on = [ + depends_on = [ aws_eks_cluster.this, - kubernetes_config_map.aws_auth, aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy, aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy, aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly From b876ff95136fbb419cbb33feaa8f354a053047e0 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 22 Nov 2021 11:15:50 -0500 Subject: [PATCH 420/962] fix: update CI/CD process to enable auto-release workflow (#1698) --- .chglog/CHANGELOG.tpl.md | 75 --- .chglog/config.yml | 54 -- .../CHANGELOG.pre-v11.0.0.md | 0 .github/ISSUE_TEMPLATE.md | 30 - .github/PULL_REQUEST_TEMPLATE.md | 9 - .github/workflows/changelog-check.yaml | 36 - .github/workflows/pre-commit.yml | 103 ++- .github/workflows/release.yml | 32 + .pre-commit-config.yaml | 6 +- .releaserc.json | 36 + CHANGELOG.md | 12 - Makefile | 17 - README.md | 8 +- examples/bottlerocket/README.md | 8 +- examples/bottlerocket/versions.tf | 25 +- examples/complete/README.md | 14 +- examples/complete/versions.tf | 20 +- examples/fargate/README.md | 11 +- examples/fargate/versions.tf | 24 +- examples/instance_refresh/README.md | 12 +- examples/instance_refresh/versions.tf | 25 +- examples/irsa/README.md | 12 +- examples/irsa/versions.tf | 25 +- examples/launch_templates/README.md | 8 +- examples/launch_templates/versions.tf | 21 +- .../README.md | 8 +- .../versions.tf | 20 +- examples/managed_node_groups/README.md | 8 +- examples/managed_node_groups/versions.tf | 20 +- examples/secrets_encryption/README.md | 8 +- examples/secrets_encryption/versions.tf | 20 +- modules/fargate/README.md | 4 +- modules/fargate/versions.tf | 5 +- modules/node_groups/README.md | 4 +- modules/node_groups/versions.tf | 10 +- tools/semtag | 627 ------------------ versions.tf | 20 +- 37 files changed, 354 insertions(+), 1023 deletions(-) delete mode 100644 .chglog/CHANGELOG.tpl.md delete mode 100644 .chglog/config.yml rename CHANGELOG.pre-v11.0.0.md => .github/CHANGELOG.pre-v11.0.0.md (100%) delete mode 100644 .github/ISSUE_TEMPLATE.md delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 .github/workflows/changelog-check.yaml create mode 100644 .github/workflows/release.yml create mode 100644 .releaserc.json delete mode 100644 Makefile delete mode 100755 tools/semtag diff --git a/.chglog/CHANGELOG.tpl.md b/.chglog/CHANGELOG.tpl.md deleted file mode 100644 index 1af11b8438..0000000000 --- a/.chglog/CHANGELOG.tpl.md +++ /dev/null @@ -1,75 +0,0 @@ -# Change Log - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/) and this -project adheres to [Semantic Versioning](http://semver.org/). - -{{ if .Versions -}} - -## [Unreleased] -{{ if .Unreleased.CommitGroups -}} -{{ range .Unreleased.CommitGroups -}} -{{ .Title }}: -{{ range .Commits -}} -{{- if .Subject -}} -- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject | upperFirst }} -{{ end -}} -{{ end }} -{{ end -}} -{{ else }} -{{ range .Unreleased.Commits -}} -{{- if .Subject -}} -- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject | upperFirst}} -{{ end -}} -{{ end }} -{{ end -}} - -{{- if .Unreleased.NoteGroups -}} -{{ range .Unreleased.NoteGroups -}} -{{ .Title }}: -{{ range .Notes -}} -- {{ .Body }} -{{ end }} -{{ end -}} -{{ end -}} -{{ end -}} - -{{ range .Versions }} - -## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }} -{{ if .CommitGroups -}} -{{ range .CommitGroups -}} -{{ .Title }}: -{{ range .Commits -}} -{{- if .Subject -}} -- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject | upperFirst }} -{{ end -}} -{{ end }} -{{ end -}} -{{ else }} -{{ range .Commits -}} -{{- if .Subject -}} -- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject | upperFirst }} -{{ end -}} -{{ end }} -{{ end -}} - -{{- if .NoteGroups -}} -{{ range .NoteGroups -}} -{{ .Title }}: -{{ range .Notes -}} -- {{ .Body }} -{{ end }} -{{ end -}} -{{ end -}} -{{ end -}} - -{{- if .Versions }} -[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD -{{ range .Versions -}} -{{ if .Tag.Previous -}} -[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }} -{{ end -}} -{{ end -}} -{{ end -}} diff --git a/.chglog/config.yml b/.chglog/config.yml deleted file mode 100644 index 06f2cfb375..0000000000 --- a/.chglog/config.yml +++ /dev/null @@ -1,54 +0,0 @@ -style: github -template: CHANGELOG.tpl.md -info: - title: CHANGELOG - repository_url: https://github.com/terraform-aws-modules/terraform-aws-eks -options: - commits: - sort_by: Type - filters: - Type: - - feat - - fix - - improvement - - docs - - refactor - - test - - ci - - commit_groups: - group_by: Type - sort_by: Custom - title_order: - - feat - - improvement - - refactor - - fix - - docs - - test - - ci - title_maps: - feat: FEATURES - fix: BUG FIXES - improvement: ENHANCEMENTS - docs: DOCS - refactor: REFACTORS - test: TESTS - ci: CI - - header: - pattern: "^(.+)\\s*:\\s*(.+)$" - pattern_maps: - - Type - - Subject - - notes: - keywords: - - BREAKING CHANGES - - NOTES - - refs: - actions: - - Closes - - Fixes - - Resolves diff --git a/CHANGELOG.pre-v11.0.0.md b/.github/CHANGELOG.pre-v11.0.0.md similarity index 100% rename from CHANGELOG.pre-v11.0.0.md rename to .github/CHANGELOG.pre-v11.0.0.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index bcb7188d94..0000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,30 +0,0 @@ -# I have issues - -## I'm submitting a... - -* [ ] bug report -* [ ] feature request -* [ ] support request - read the [FAQ](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md) first! -* [ ] kudos, thank you, warm fuzzy - -## What is the current behavior? - - - -## If this is a bug, how to reproduce? Please include a code sample if relevant. - - - -## What's the expected behavior? - - - -## Are you able to fix this problem and submit a PR? Link here if you have already. - -## Environment details - -* Affected module version: -* OS: -* Terraform version: - -## Any other relevant info diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index ce48f0220a..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,9 +0,0 @@ -# PR o'clock - -## Description - -Please explain the changes you made here and link to any relevant issues. - -### Checklist - -- [ ] README.md has been updated after any changes to variables and outputs. See https://github.com/terraform-aws-modules/terraform-aws-eks/#doc-generation diff --git a/.github/workflows/changelog-check.yaml b/.github/workflows/changelog-check.yaml deleted file mode 100644 index 0eee4fdbe0..0000000000 --- a/.github/workflows/changelog-check.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: CHANGELOG Checks -on: - pull_request_target: - paths: - - CHANGELOG.md - branches-ignore: - - releases/** - -jobs: - comment: - name: Comment - runs-on: ubuntu-latest - steps: - - name: Find Existing PR Comment - id: prc - uses: peter-evans/find-comment@v1 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: "github-actions[bot]" - body-includes: "The `CHANGELOG.md` file contents are handled by the maintainers during merge." - - name: PR Comment - if: ${{ steps.prc.outputs.comment-id == '' }} - uses: peter-evans/create-or-update-comment@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - issue-number: ${{ github.event.pull_request.number }} - body: | - Thank you for your contribution! - - The `CHANGELOG.md` file contents are handled by the maintainers during merge. This is to prevent pull request merge conflicts. - Please see the Contributing Guide for additional pull request review items. - - Remove any changes to the `CHANGELOG.md` file and commit them in this pull request. - - name: Fail the check if changelog change - run: exit 1 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 5a3c0ce7e1..b8f1b8a5ab 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -2,98 +2,77 @@ name: Pre-Commit on: pull_request: - push: branches: + - main - master +env: + TERRAFORM_DOCS_VERSION: v0.16.0 + jobs: - # Min Terraform version(s) - getDirectories: - name: Get root directories + collectInputs: + name: Collect workflow inputs runs-on: ubuntu-latest + outputs: + directories: ${{ steps.dirs.outputs.directories }} steps: - name: Checkout uses: actions/checkout@v2 - - name: Install Python - uses: actions/setup-python@v2 - - name: Build matrix - id: matrix - run: | - DIRS=$(python -c "import json; import glob; print(json.dumps([x.replace('/versions.tf', '') for x in glob.glob('./**/versions.tf', recursive=True)]))") - echo "::set-output name=directories::$DIRS" - outputs: - directories: ${{ steps.matrix.outputs.directories }} + + - name: Get root directories + id: dirs + uses: clowdhaus/terraform-composite-actions/directories@v1.3.0 preCommitMinVersions: - name: Min TF validate - needs: getDirectories + name: Min TF pre-commit + needs: collectInputs runs-on: ubuntu-latest strategy: matrix: - directory: ${{ fromJson(needs.getDirectories.outputs.directories) }} + directory: ${{ fromJson(needs.collectInputs.outputs.directories) }} steps: - name: Checkout uses: actions/checkout@v2 - - name: Install Python - uses: actions/setup-python@v2 + - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.0.2 + uses: clowdhaus/terraform-min-max@v1.0.3 with: directory: ${{ matrix.directory }} - - name: Install Terraform v${{ steps.minMax.outputs.minVersion }} - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: ${{ steps.minMax.outputs.minVersion }} - - name: Install pre-commit dependencies - run: pip install pre-commit - - name: Execute pre-commit + + - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory != '.' }} - run: pre-commit run terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/* - - name: Execute pre-commit + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.3.0 + with: + terraform-version: ${{ steps.minMax.outputs.minVersion }} + args: 'terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/*' + + - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory == '.' }} - run: pre-commit run terraform_validate --color=always --show-diff-on-failure --files $(ls *.tf) + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.3.0 + with: + terraform-version: ${{ steps.minMax.outputs.minVersion }} + args: 'terraform_validate --color=always --show-diff-on-failure --files $(ls *.tf)' - # Max Terraform version - getBaseVersion: - name: Module max TF version + preCommitMaxVersion: + name: Max TF pre-commit runs-on: ubuntu-latest + needs: collectInputs steps: - name: Checkout uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{github.event.pull_request.head.repo.full_name}} + - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.0.2 - outputs: - minVersion: ${{ steps.minMax.outputs.minVersion }} - maxVersion: ${{ steps.minMax.outputs.maxVersion }} + uses: clowdhaus/terraform-min-max@v1.0.3 - preCommitMaxVersion: - name: Max TF pre-commit - runs-on: ubuntu-latest - needs: getBaseVersion - strategy: - fail-fast: false - matrix: - version: - - ${{ needs.getBaseVersion.outputs.maxVersion }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install Python - uses: actions/setup-python@v2 - - name: Install Terraform v${{ matrix.version }} - uses: hashicorp/setup-terraform@v1 + - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.3.0 with: - terraform_version: ${{ matrix.version }} - - name: Install pre-commit dependencies - run: | - pip install pre-commit - curl -Lo ./terraform-docs.tar.gz https://github.com/terraform-docs/terraform-docs/releases/download/v0.13.0/terraform-docs-v0.13.0-$(uname)-amd64.tar.gz && tar -xzf terraform-docs.tar.gz terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ - curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip tflint && rm tflint.zip && sudo mv tflint /usr/bin/ - - name: Execute pre-commit - # Run all pre-commit checks on max version supported - if: ${{ matrix.version == needs.getBaseVersion.outputs.maxVersion }} - run: pre-commit run --color=always --show-diff-on-failure --all-files + terraform-version: ${{ steps.minMax.outputs.maxVersion }} + terraform-docs-version: ${{ env.TERRAFORM_DOCS_VERSION }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..141937d863 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,32 @@ +name: Release + +on: + workflow_dispatch: + push: + branches: + - main + - master + paths: + - '**/*.py' + - '**/*.tf' + +jobs: + release: + name: Release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Release + uses: cycjimmy/semantic-release-action@v2 + with: + semantic_version: 18.0.0 + extra_plugins: | + @semantic-release/changelog@6.0.0 + @semantic-release/git@10.0.0 + env: + GITHUB_TOKEN: ${{ secrets.SEMANTIC_RELEASE_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d26bea60b7..20e651ab6a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,12 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.55.0 + rev: v1.58.0 hooks: - id: terraform_fmt - id: terraform_validate - id: terraform_docs + args: + - '--args=--lockfile=false' - id: terraform_tflint args: - '--args=--only=terraform_deprecated_interpolation' @@ -15,7 +17,7 @@ repos: - '--args=--only=terraform_documented_variables' - '--args=--only=terraform_typed_variables' - '--args=--only=terraform_module_pinned_source' -# - '--args=--only=terraform_naming_convention' + # - '--args=--only=terraform_naming_convention' - '--args=--only=terraform_required_version' - '--args=--only=terraform_required_providers' - '--args=--only=terraform_standard_module_structure' diff --git a/.releaserc.json b/.releaserc.json new file mode 100644 index 0000000000..6e39031cf2 --- /dev/null +++ b/.releaserc.json @@ -0,0 +1,36 @@ +{ + "branches": [ + "main", + "master" + ], + "ci": false, + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + [ + "@semantic-release/github", + { + "successComment": + "This ${issue.pull_request ? 'PR is included' : 'issue has been resolved'} in version ${nextRelease.version} :tada:", + "labels": false, + "releasedLabels": false + } + ], + [ + "@semantic-release/changelog", + { + "changelogFile": "CHANGELOG.md", + "changelogTitle": "# Changelog\n\nAll notable changes to this project will be documented in this file." + } + ], + [ + "@semantic-release/git", + { + "assets": [ + "CHANGELOG.md" + ], + "message": "chore(release): version ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" + } + ] + ] +} diff --git a/CHANGELOG.md b/CHANGELOG.md index fe6345f9b5..2153e79f74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,3 @@ -# Change Log - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/) and this -project adheres to [Semantic Versioning](http://semver.org/). - - -## [Unreleased] - - - ## [v17.23.0] - 2021-11-02 FEATURES: diff --git a/Makefile b/Makefile deleted file mode 100644 index 5120cc143a..0000000000 --- a/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -.PHONY: changelog release - -SEMTAG=tools/semtag - -CHANGELOG_FILE=CHANGELOG.md -TAG_QUERY=v11.0.0.. - -scope ?= "minor" - -changelog-unrelease: - git-chglog --no-case -o $(CHANGELOG_FILE) $(TAG_QUERY) - -changelog: - git-chglog --no-case -o $(CHANGELOG_FILE) --next-tag `$(SEMTAG) final -s $(scope) -o -f` $(TAG_QUERY) - -release: - $(SEMTAG) final -s $(scope) diff --git a/README.md b/README.md index 5980b48201..26c6f11cec 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56.0 | +| [aws](#requirement\_aws) | >= 3.56 | | [cloudinit](#requirement\_cloudinit) | >= 2.0 | | [http](#requirement\_http) | >= 2.4.1 | | [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | @@ -126,7 +126,7 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.56.0 | +| [aws](#provider\_aws) | >= 3.56 | | [http](#provider\_http) | >= 2.4.1 | | [kubernetes](#provider\_kubernetes) | >= 1.11.1 | | [local](#provider\_local) | >= 1.4 | @@ -135,8 +135,8 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | Name | Source | Version | |------|--------|---------| -| [fargate](#module\_fargate) | ./modules/fargate | | -| [node\_groups](#module\_node\_groups) | ./modules/node_groups | | +| [fargate](#module\_fargate) | ./modules/fargate | n/a | +| [node\_groups](#module\_node\_groups) | ./modules/node_groups | n/a | ## Resources diff --git a/examples/bottlerocket/README.md b/examples/bottlerocket/README.md index e02687da98..88d9d4ee14 100644 --- a/examples/bottlerocket/README.md +++ b/examples/bottlerocket/README.md @@ -24,8 +24,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [aws](#requirement\_aws) | >= 3.56 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | | [tls](#requirement\_tls) | >= 2.0 | @@ -34,7 +34,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | +| [aws](#provider\_aws) | >= 3.56 | | [random](#provider\_random) | >= 2.1 | | [tls](#provider\_tls) | >= 2.0 | @@ -42,7 +42,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| -| [eks](#module\_eks) | ../.. | | +| [eks](#module\_eks) | ../.. | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources diff --git a/examples/bottlerocket/versions.tf b/examples/bottlerocket/versions.tf index 6adb95785f..cb5115c487 100644 --- a/examples/bottlerocket/versions.tf +++ b/examples/bottlerocket/versions.tf @@ -2,10 +2,25 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" - local = ">= 1.4" - random = ">= 2.1" - kubernetes = "~> 2.0" - tls = ">= 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + local = { + source = "hashicorp/local" + version = ">= 1.4" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 1.11.1" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" + } + tls = { + source = "hashicorp/tls" + version = ">= 2.0" + } } } diff --git a/examples/complete/README.md b/examples/complete/README.md index 3fbaf99418..5cf7785cb1 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -24,8 +24,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [aws](#requirement\_aws) | >= 3.56 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -33,17 +33,17 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | +| [aws](#provider\_aws) | >= 3.56 | | [random](#provider\_random) | >= 2.1 | ## Modules | Name | Source | Version | |------|--------|---------| -| [disabled\_eks](#module\_disabled\_eks) | ../.. | | -| [disabled\_fargate](#module\_disabled\_fargate) | ../../modules/fargate | | -| [disabled\_node\_groups](#module\_disabled\_node\_groups) | ../../modules/node_groups | | -| [eks](#module\_eks) | ../.. | | +| [disabled\_eks](#module\_disabled\_eks) | ../.. | n/a | +| [disabled\_fargate](#module\_disabled\_fargate) | ../../modules/fargate | n/a | +| [disabled\_node\_groups](#module\_disabled\_node\_groups) | ../../modules/node_groups | n/a | +| [eks](#module\_eks) | ../.. | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index bbcf893252..8e2b837984 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -2,9 +2,21 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" - local = ">= 1.4" - random = ">= 2.1" - kubernetes = "~> 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + local = { + source = "hashicorp/local" + version = ">= 1.4" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 1.11.1" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" + } } } diff --git a/examples/fargate/README.md b/examples/fargate/README.md index b4e79abdb7..1228f5c4c1 100644 --- a/examples/fargate/README.md +++ b/examples/fargate/README.md @@ -23,24 +23,25 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [aws](#requirement\_aws) | >= 3.56 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | +| [tls](#requirement\_tls) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | +| [aws](#provider\_aws) | >= 3.56 | | [random](#provider\_random) | >= 2.1 | ## Modules | Name | Source | Version | |------|--------|---------| -| [eks](#module\_eks) | ../.. | | -| [fargate\_profile\_existing\_cluster](#module\_fargate\_profile\_existing\_cluster) | ../../modules/fargate | | +| [eks](#module\_eks) | ../.. | n/a | +| [fargate\_profile\_existing\_cluster](#module\_fargate\_profile\_existing\_cluster) | ../../modules/fargate | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources diff --git a/examples/fargate/versions.tf b/examples/fargate/versions.tf index bbcf893252..cb5115c487 100644 --- a/examples/fargate/versions.tf +++ b/examples/fargate/versions.tf @@ -2,9 +2,25 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" - local = ">= 1.4" - random = ">= 2.1" - kubernetes = "~> 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + local = { + source = "hashicorp/local" + version = ">= 1.4" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 1.11.1" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" + } + tls = { + source = "hashicorp/tls" + version = ">= 2.0" + } } } diff --git a/examples/instance_refresh/README.md b/examples/instance_refresh/README.md index 73b836d689..cc558693ae 100644 --- a/examples/instance_refresh/README.md +++ b/examples/instance_refresh/README.md @@ -22,9 +22,9 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | -| [helm](#requirement\_helm) | ~> 2.0 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [aws](#requirement\_aws) | >= 3.56 | +| [helm](#requirement\_helm) | >= 2.0 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -32,8 +32,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | -| [helm](#provider\_helm) | ~> 2.0 | +| [aws](#provider\_aws) | >= 3.56 | +| [helm](#provider\_helm) | >= 2.0 | | [random](#provider\_random) | >= 2.1 | ## Modules @@ -42,7 +42,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|--------|---------| | [aws\_node\_termination\_handler\_role](#module\_aws\_node\_termination\_handler\_role) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | 4.1.0 | | [aws\_node\_termination\_handler\_sqs](#module\_aws\_node\_termination\_handler\_sqs) | terraform-aws-modules/sqs/aws | ~> 3.0.0 | -| [eks](#module\_eks) | ../.. | | +| [eks](#module\_eks) | ../.. | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources diff --git a/examples/instance_refresh/versions.tf b/examples/instance_refresh/versions.tf index f546ca0cf0..67c2d66a82 100644 --- a/examples/instance_refresh/versions.tf +++ b/examples/instance_refresh/versions.tf @@ -2,10 +2,25 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" - local = ">= 1.4" - random = ">= 2.1" - kubernetes = "~> 2.0" - helm = "~> 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + local = { + source = "hashicorp/local" + version = ">= 1.4" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 1.11.1" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.0" + } } } diff --git a/examples/irsa/README.md b/examples/irsa/README.md index 770ca3f902..137f3d63d8 100644 --- a/examples/irsa/README.md +++ b/examples/irsa/README.md @@ -22,9 +22,9 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | -| [helm](#requirement\_helm) | ~> 2.0 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [aws](#requirement\_aws) | >= 3.56 | +| [helm](#requirement\_helm) | >= 2.0 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -32,15 +32,15 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | -| [helm](#provider\_helm) | ~> 2.0 | +| [aws](#provider\_aws) | >= 3.56 | +| [helm](#provider\_helm) | >= 2.0 | | [random](#provider\_random) | >= 2.1 | ## Modules | Name | Source | Version | |------|--------|---------| -| [eks](#module\_eks) | ../.. | | +| [eks](#module\_eks) | ../.. | n/a | | [iam\_assumable\_role\_admin](#module\_iam\_assumable\_role\_admin) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | ~> 4.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | diff --git a/examples/irsa/versions.tf b/examples/irsa/versions.tf index f546ca0cf0..67c2d66a82 100644 --- a/examples/irsa/versions.tf +++ b/examples/irsa/versions.tf @@ -2,10 +2,25 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" - local = ">= 1.4" - random = ">= 2.1" - kubernetes = "~> 2.0" - helm = "~> 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + local = { + source = "hashicorp/local" + version = ">= 1.4" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 1.11.1" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.0" + } } } diff --git a/examples/launch_templates/README.md b/examples/launch_templates/README.md index 6d1d5e2549..dbd5073f89 100644 --- a/examples/launch_templates/README.md +++ b/examples/launch_templates/README.md @@ -22,8 +22,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [aws](#requirement\_aws) | >= 3.56 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -31,14 +31,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | +| [aws](#provider\_aws) | >= 3.56 | | [random](#provider\_random) | >= 2.1 | ## Modules | Name | Source | Version | |------|--------|---------| -| [eks](#module\_eks) | ../.. | | +| [eks](#module\_eks) | ../.. | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources diff --git a/examples/launch_templates/versions.tf b/examples/launch_templates/versions.tf index 9c1dbfa3e8..8e2b837984 100644 --- a/examples/launch_templates/versions.tf +++ b/examples/launch_templates/versions.tf @@ -2,10 +2,21 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" - local = ">= 1.4" - random = ">= 2.1" - kubernetes = "~> 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + local = { + source = "hashicorp/local" + version = ">= 1.4" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 1.11.1" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" + } } } - diff --git a/examples/launch_templates_with_managed_node_groups/README.md b/examples/launch_templates_with_managed_node_groups/README.md index e79ff558f9..3cae549325 100644 --- a/examples/launch_templates_with_managed_node_groups/README.md +++ b/examples/launch_templates_with_managed_node_groups/README.md @@ -25,8 +25,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [aws](#requirement\_aws) | >= 3.56 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -34,14 +34,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | +| [aws](#provider\_aws) | >= 3.56 | | [random](#provider\_random) | >= 2.1 | ## Modules | Name | Source | Version | |------|--------|---------| -| [eks](#module\_eks) | ../.. | | +| [eks](#module\_eks) | ../.. | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources diff --git a/examples/launch_templates_with_managed_node_groups/versions.tf b/examples/launch_templates_with_managed_node_groups/versions.tf index bbcf893252..8e2b837984 100644 --- a/examples/launch_templates_with_managed_node_groups/versions.tf +++ b/examples/launch_templates_with_managed_node_groups/versions.tf @@ -2,9 +2,21 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" - local = ">= 1.4" - random = ">= 2.1" - kubernetes = "~> 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + local = { + source = "hashicorp/local" + version = ">= 1.4" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 1.11.1" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" + } } } diff --git a/examples/managed_node_groups/README.md b/examples/managed_node_groups/README.md index 8cc2ecd8ae..7121431392 100644 --- a/examples/managed_node_groups/README.md +++ b/examples/managed_node_groups/README.md @@ -25,8 +25,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [aws](#requirement\_aws) | >= 3.56 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -34,14 +34,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | +| [aws](#provider\_aws) | >= 3.56 | | [random](#provider\_random) | >= 2.1 | ## Modules | Name | Source | Version | |------|--------|---------| -| [eks](#module\_eks) | ../.. | | +| [eks](#module\_eks) | ../.. | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources diff --git a/examples/managed_node_groups/versions.tf b/examples/managed_node_groups/versions.tf index bbcf893252..8e2b837984 100644 --- a/examples/managed_node_groups/versions.tf +++ b/examples/managed_node_groups/versions.tf @@ -2,9 +2,21 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" - local = ">= 1.4" - random = ">= 2.1" - kubernetes = "~> 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + local = { + source = "hashicorp/local" + version = ">= 1.4" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 1.11.1" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" + } } } diff --git a/examples/secrets_encryption/README.md b/examples/secrets_encryption/README.md index c3ee731f2a..f5f38b0498 100644 --- a/examples/secrets_encryption/README.md +++ b/examples/secrets_encryption/README.md @@ -22,8 +22,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.22.0 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [aws](#requirement\_aws) | >= 3.56 | +| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | | [local](#requirement\_local) | >= 1.4 | | [random](#requirement\_random) | >= 2.1 | @@ -31,14 +31,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.22.0 | +| [aws](#provider\_aws) | >= 3.56 | | [random](#provider\_random) | >= 2.1 | ## Modules | Name | Source | Version | |------|--------|---------| -| [eks](#module\_eks) | ../.. | | +| [eks](#module\_eks) | ../.. | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources diff --git a/examples/secrets_encryption/versions.tf b/examples/secrets_encryption/versions.tf index bbcf893252..8e2b837984 100644 --- a/examples/secrets_encryption/versions.tf +++ b/examples/secrets_encryption/versions.tf @@ -2,9 +2,21 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.22.0" - local = ">= 1.4" - random = ">= 2.1" - kubernetes = "~> 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + local = { + source = "hashicorp/local" + version = ">= 1.4" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 1.11.1" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" + } } } diff --git a/modules/fargate/README.md b/modules/fargate/README.md index 466b2a051a..7a20be175b 100644 --- a/modules/fargate/README.md +++ b/modules/fargate/README.md @@ -24,13 +24,13 @@ See example code in `examples/fargate`. | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.40.0 | +| [aws](#requirement\_aws) | >= 3.56 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.40.0 | +| [aws](#provider\_aws) | >= 3.56 | ## Modules diff --git a/modules/fargate/versions.tf b/modules/fargate/versions.tf index 2051547e51..9480a77da8 100644 --- a/modules/fargate/versions.tf +++ b/modules/fargate/versions.tf @@ -2,6 +2,9 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.40.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } } } diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index e0bf4f3180..f91ce04ae0 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -65,14 +65,14 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56.0 | +| [aws](#requirement\_aws) | >= 3.56 | | [cloudinit](#requirement\_cloudinit) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.56.0 | +| [aws](#provider\_aws) | >= 3.56 | | [cloudinit](#provider\_cloudinit) | >= 2.0 | ## Modules diff --git a/modules/node_groups/versions.tf b/modules/node_groups/versions.tf index 5324b482ab..f4d0036a86 100644 --- a/modules/node_groups/versions.tf +++ b/modules/node_groups/versions.tf @@ -2,7 +2,13 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.56.0" - cloudinit = ">= 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + cloudinit = { + source = "hashicorp/cloudinit" + version = ">= 2.0" + } } } diff --git a/tools/semtag b/tools/semtag deleted file mode 100755 index 568d4241ad..0000000000 --- a/tools/semtag +++ /dev/null @@ -1,627 +0,0 @@ -#!/usr/bin/env bash -# -# Thanks to @pnikosis for this script https://github.com/pnikosis/semtag -# -PROG=semtag -PROG_VERSION="v0.1.0" - -SEMVER_REGEX="^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(\-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$" -IDENTIFIER_REGEX="^\-([0-9A-Za-z-]+)\.([0-9A-Za-z-]+)*$" - -# Global variables -FIRST_VERSION="v0.0.0" -finalversion=$FIRST_VERSION -lastversion=$FIRST_VERSION -hasversiontag="false" -scope="patch" -displayonly="false" -forcetag="false" -forcedversion= -versionname= -identifier= - -HELP="\ -Usage: - $PROG - $PROG getlast - $PROG getfinal - $PROG (final|alpha|beta|candidate) [-s (major|minor|patch|auto) | -o] - $PROG --help - $PROG --version -Options: - -s The scope that must be increased, can be major, minor or patch. - The resulting version will match X.Y.Z(-PRERELEASE)(+BUILD) - where X, Y and Z are positive integers, PRERELEASE is an optionnal - string composed of alphanumeric characters describing if the build is - a release candidate, alpha or beta version, with a number. - BUILD is also an optional string composed of alphanumeric - characters and hyphens. - Setting the scope as 'auto', the script will chose the scope between - 'minor' and 'patch', depending on the amount of lines added (<10% will - choose patch). - -v Specifies manually the version to be tagged, must be a valid semantic version - in the format X.Y.Z where X, Y and Z are positive integers. - -o Output the version only, shows the bumped version, but doesn't tag. - -f Forces to tag, even if there are unstaged or uncommited changes. -Commands: - --help Print this help message. - --version Prints the program's version. - get Returns both current final version and last tagged version. - getlast Returns the latest tagged version. - getfinal Returns the latest tagged final version. - getcurrent Returns the current version, based on the latest one, if there are uncommited or - unstaged changes, they will be reflected in the version, adding the number of - pending commits, current branch and commit hash. - final Tags the current build as a final version, this only can be done on the master branch. - candidate Tags the current build as a release candidate, the tag will contain all - the commits from the last final version. - alpha Tags the current build as an alpha version, the tag will contain all - the commits from the last final version. - beta Tags the current build as a beta version, the tag will contain all - the commits from the last final version." - -# Commands and options -ACTION="getlast" -ACTION="$1" -shift - -# We get the parameters -while getopts "v:s:of" opt; do - case $opt in - v) - forcedversion="$OPTARG" - ;; - s) - scope="$OPTARG" - ;; - o) - displayonly="true" - ;; - f) - forcetag="true" - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "Option -$OPTARG requires an argument." >&2 - exit 1 - ;; - esac -done - -# Gets a string with the version and returns an array of maximum size of 5 with all the parts of the sematinc version -# $1 The string containing the version in semantic format -# $2 The variable to store the result array: -# position 0: major number -# position 1: minor number -# position 2: patch number -# position 3: identifier (or prerelease identifier) -# position 4: build info -function explode_version { - local __version=$1 - local __result=$2 - if [[ $__version =~ $SEMVER_REGEX ]] ; then - local __major=${BASH_REMATCH[1]} - local __minor=${BASH_REMATCH[2]} - local __patch=${BASH_REMATCH[3]} - local __prere=${BASH_REMATCH[4]} - local __build=${BASH_REMATCH[5]} - eval "$__result=(\"$__major\" \"$__minor\" \"$__patch\" \"$__prere\" \"$__build\")" - else - eval "$__result=" - fi -} - -# Compare two versions and returns -1, 0 or 1 -# $1 The first version to compare -# $2 The second version to compare -# $3 The variable where to store the result -function compare_versions { - local __first - local __second - explode_version $1 __first - explode_version $2 __second - local lv=$3 - - # Compares MAJOR, MINOR and PATCH - for i in 0 1 2; do - local __numberfirst=${__first[$i]} - local __numbersecond=${__second[$i]} - case $(($__numberfirst - $__numbersecond)) in - 0) - ;; - -[0-9]*) - eval "$lv=-1" - return 0 - ;; - [0-9]*) - eval "$lv=1" - return 0 - ;; - esac - done - - # Identifiers should compare with the ASCII order. - local __identifierfirst=${__first[3]} - local __identifiersecond=${__second[3]} - if [[ -n "$__identifierfirst" ]] && [[ -n "$__identifiersecond" ]]; then - if [[ "$__identifierfirst" > "$__identifiersecond" ]]; then - eval "$lv=1" - return 0 - elif [[ "$__identifierfirst" < "$__identifiersecond" ]]; then - eval "$lv=-1" - return 0 - fi - elif [[ -z "$__identifierfirst" ]] && [[ -n "$__identifiersecond" ]]; then - eval "$lv=1" - return 0 - elif [[ -n "$__identifierfirst" ]] && [[ -z "$__identifiersecond" ]]; then - eval "$lv=-1" - return 0 - fi - - eval "$lv=0" -} - -# Returns the last version of two -# $1 The first version to compare -# $2 The second version to compare -# $3 The variable where to store the last one -function get_latest_of_two { - local __first=$1 - local __second=$2 - local __result - local __latest=$3 - compare_versions $__first $__second __result - case $__result in - 0) - eval "$__latest=$__second" - ;; - -1) - eval "$__latest=$__second" - ;; - 1) - eval "$__latest=$__first" - ;; - esac -} - -# Assigns a 2 size array with the identifier, having the identifier at pos 0, and the number in pos 1 -# $1 The identifier in the format -id.# -# $2 The vferiable where to store the 2 size array -function explode_identifier { - local __identifier=$1 - local __result=$2 - if [[ $__identifier =~ $IDENTIFIER_REGEX ]] ; then - local __id=${BASH_REMATCH[1]} - local __number=${BASH_REMATCH[2]} - if [[ -z "$__number" ]]; then - __number=1 - fi - eval "$__result=(\"$__id\" \"$__number\")" - else - eval "$__result=" - fi -} - -# Gets a list of tags and assigns the base and latest versions -# Receives an array with the tags containing the versions -# Assigns to the global variables finalversion and lastversion the final version and the latest version -function get_latest { - local __taglist=("$@") - local __tagsnumber=${#__taglist[@]} - local __current - case $__tagsnumber in - 0) - finalversion=$FIRST_VERSION - lastversion=$FIRST_VERSION - ;; - 1) - __current=${__taglist[0]} - explode_version $__current ver - if [ -n "$ver" ]; then - if [ -n "${ver[3]}" ]; then - finalversion=$FIRST_VERSION - else - finalversion=$__current - fi - lastversion=$__current - else - finalversion=$FIRST_VERSION - lastversion=$FIRST_VERSION - fi - ;; - *) - local __lastpos=$(($__tagsnumber-1)) - for i in $(seq 0 $__lastpos) - do - __current=${__taglist[i]} - explode_version ${__taglist[i]} ver - if [ -n "$ver" ]; then - if [ -z "${ver[3]}" ]; then - get_latest_of_two $finalversion $__current finalversion - get_latest_of_two $lastversion $finalversion lastversion - else - get_latest_of_two $lastversion $__current lastversion - fi - fi - done - ;; - esac - - if git rev-parse -q --verify "refs/tags/$lastversion" >/dev/null; then - hasversiontag="true" - else - hasversiontag="false" - fi -} - -# Gets the next version given the provided scope -# $1 The version that is going to be bumped -# $2 The scope to bump -# $3 The variable where to stoer the result -function get_next_version { - local __exploded - local __fromversion=$1 - local __scope=$2 - local __result=$3 - explode_version $__fromversion __exploded - case $__scope in - major) - __exploded[0]=$((${__exploded[0]}+1)) - __exploded[1]=0 - __exploded[2]=0 - ;; - minor) - __exploded[1]=$((${__exploded[1]}+1)) - __exploded[2]=0 - ;; - patch) - __exploded[2]=$((${__exploded[2]}+1)) - ;; - esac - - eval "$__result=v${__exploded[0]}.${__exploded[1]}.${__exploded[2]}" -} - -function bump_version { - ## First we try to get the next version based on the existing last one - if [ "$scope" == "auto" ]; then - get_scope_auto scope - fi - - local __candidatefromlast=$FIRST_VERSION - local __explodedlast - explode_version $lastversion __explodedlast - if [[ -n "${__explodedlast[3]}" ]]; then - # Last version is not final - local __idlast - explode_identifier ${__explodedlast[3]} __idlast - - # We get the last, given the desired id based on the scope - __candidatefromlast="v${__explodedlast[0]}.${__explodedlast[1]}.${__explodedlast[2]}" - if [[ -n "$identifier" ]]; then - local __nextid="$identifier.1" - if [ "$identifier" == "${__idlast[0]}" ]; then - # We target the same identifier as the last so we increase one - __nextid="$identifier.$(( ${__idlast[1]}+1 ))" - __candidatefromlast="$__candidatefromlast-$__nextid" - else - # Different identifiers, we make sure we are assigning a higher identifier, if not, we increase the version - __candidatefromlast="$__candidatefromlast-$__nextid" - local __comparedwithlast - compare_versions $__candidatefromlast $lastversion __comparedwithlast - if [ "$__comparedwithlast" == -1 ]; then - get_next_version $__candidatefromlast $scope __candidatefromlast - __candidatefromlast="$__candidatefromlast-$__nextid" - fi - fi - fi - fi - - # Then we try to get the version based on the latest final one - local __candidatefromfinal=$FIRST_VERSION - get_next_version $finalversion $scope __candidatefromfinal - if [[ -n "$identifier" ]]; then - __candidatefromfinal="$__candidatefromfinal-$identifier.1" - fi - - # Finally we compare both candidates - local __resultversion - local __result - compare_versions $__candidatefromlast $__candidatefromfinal __result - case $__result in - 0) - __resultversion=$__candidatefromlast - ;; - -1) - __resultversion="$__candidatefromfinal" - ;; - 1) - __resultversion=$__candidatefromlast - ;; - esac - - eval "$1=$__resultversion" -} - -function increase_version { - local __version= - - if [ -z $forcedversion ]; then - bump_version __version - else - if [[ $forcedversion =~ $SEMVER_REGEX ]] ; then - compare_versions $forcedversion $lastversion __result - if [ $__result -le 0 ]; then - echo "Version can't be lower than last version: $lastversion" - exit 1 - fi - else - echo "Non valid version to bump" - exit 1 - fi - __version=$forcedversion - fi - - if [ "$displayonly" == "true" ]; then - echo "$__version" - else - if [ "$forcetag" == "false" ]; then - check_git_dirty_status - fi - local __commitlist - if [ "$finalversion" == "$FIRST_VERSION" ] || [ "$hasversiontag" != "true" ]; then - __commitlist="$(git log --pretty=oneline | cat)" - else - __commitlist="$(git log --pretty=oneline $finalversion... | cat)" - fi - - # If we are forcing a bump, we add bump to the commit list - if [[ -z $__commitlist && "$forcetag" == "true" ]]; then - __commitlist="bump" - fi - - if [[ -z $__commitlist ]]; then - echo "No commits since the last final version, not bumping version" - else - if [[ -z $versionname ]]; then - versionname=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - fi - local __message="$versionname -$__commitlist" - - # We check we have info on the user - local __username=$(git config user.name) - if [ -z "$__username" ]; then - __username=$(id -u -n) - git config user.name $__username - fi - local __useremail=$(git config user.email) - if [ -z "$__useremail" ]; then - __useremail=$(hostname) - git config user.email "$__username@$__useremail" - fi - - git tag -a $__version -m "$__message" - - # If we have a remote, we push there - local __remotes=$(git remote) - if [[ -n $__remotes ]]; then - for __remote in $__remotes; do - git push $__remote $__version > /dev/null - if [ $? -eq 0 ]; then - echo "$__version pushed to $__remote" - else - echo "Error pushing the tag $__version to $__remote" - exit 1 - fi - done - else - echo "$__version" - fi - fi - fi -} - -function check_git_dirty_status { - local __repostatus= - get_work_tree_status __repostatus - - if [ "$__repostatus" == "uncommitted" ]; then - echo "ERROR: You have uncommitted changes" - git status --porcelain - exit 1 - fi - - if [ "$__repostatus" == "unstaged" ]; then - echo "ERROR: You have unstaged changes" - git status --porcelain - exit 1 - fi -} - -# Get the total amount of lines of code in the repo -function get_total_lines { - local __empty_id="$(git hash-object -t tree /dev/null)" - local __changes="$(git diff --numstat $__empty_id | cat)" - local __added_deleted=$1 - get_changed_lines "$__changes" $__added_deleted -} - -# Get the total amount of lines of code since the provided tag -function get_sincetag_lines { - local __sincetag=$1 - local __changes="$(git diff --numstat $__sincetag | cat)" - local __added_deleted=$2 - get_changed_lines "$__changes" $__added_deleted -} - -function get_changed_lines { - local __changes_numstat=$1 - local __result=$2 - IFS=$'\n' read -rd '' -a __changes_array <<<"$__changes_numstat" - local __diff_regex="^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+.+$" - - local __total_added=0 - local __total_deleted=0 - for i in "${__changes_array[@]}" - do - if [[ $i =~ $__diff_regex ]] ; then - local __added=${BASH_REMATCH[1]} - local __deleted=${BASH_REMATCH[2]} - __total_added=$(( $__total_added+$__added )) - __total_deleted=$(( $__total_deleted+$__deleted )) - fi - done - eval "$2=( $__total_added $__total_deleted )" -} - -function get_scope_auto { - local __verbose=$2 - local __total=0 - local __since=0 - local __scope= - - get_total_lines __total - get_sincetag_lines $finalversion __since - - local __percentage=0 - if [ "$__total" != "0" ]; then - local __percentage=$(( 100*$__since/$__total )) - if [ $__percentage -gt "10" ]; then - __scope="minor" - else - __scope="patch" - fi - fi - - eval "$1=$__scope" - if [[ -n "$__verbose" ]]; then - echo "[Auto Scope] Percentage of lines changed: $__percentage" - echo "[Auto Scope] : $__scope" - fi -} - -function get_work_tree_status { - # Update the index - git update-index -q --ignore-submodules --refresh > /dev/null - eval "$1=" - - if ! git diff-files --quiet --ignore-submodules -- > /dev/null - then - eval "$1=unstaged" - fi - - if ! git diff-index --cached --quiet HEAD --ignore-submodules -- > /dev/null - then - eval "$1=uncommitted" - fi -} - -function get_current { - if [ "$hasversiontag" == "true" ]; then - local __commitcount="$(git rev-list $lastversion.. --count)" - else - local __commitcount="$(git rev-list --count HEAD)" - fi - local __status= - get_work_tree_status __status - - if [ "$__commitcount" == "0" ] && [ -z "$__status" ]; then - eval "$1=$lastversion" - else - local __buildinfo="$(git rev-parse --short HEAD)" - local __currentbranch="$(git rev-parse --abbrev-ref HEAD)" - if [ "$__currentbranch" != "master" ]; then - __buildinfo="$__currentbranch.$__buildinfo" - fi - - local __suffix= - if [ "$__commitcount" != "0" ]; then - if [ -n "$__suffix" ]; then - __suffix="$__suffix." - fi - __suffix="$__suffix$__commitcount" - fi - if [ -n "$__status" ]; then - if [ -n "$__suffix" ]; then - __suffix="$__suffix." - fi - __suffix="$__suffix$__status" - fi - - __suffix="$__suffix+$__buildinfo" - if [ "$lastversion" == "$finalversion" ]; then - scope="patch" - identifier= - local __bumped= - bump_version __bumped - eval "$1=$__bumped-dev.$__suffix" - else - eval "$1=$lastversion.$__suffix" - fi - fi -} - -function init { - git fetch > /dev/null - TAGS="$(git tag)" - IFS=$'\n' read -rd '' -a TAG_ARRAY <<<"$TAGS" - - get_latest ${TAG_ARRAY[@]} - currentbranch="$(git rev-parse --abbrev-ref HEAD)" -} - -case $ACTION in - --help) - echo -e "$HELP" - ;; - --version) - echo -e "${PROG}: $PROG_VERSION" - ;; - final) - init - diff=$(git diff master | cat) - if [ "$forcetag" == "false" ]; then - if [ -n "$diff" ]; then - echo "ERROR: Branch must be updated with master for final versions" - exit 1 - fi - fi - increase_version - ;; - alpha|beta) - init - identifier="$ACTION" - increase_version - ;; - candidate) - init - identifier="rc" - increase_version - ;; - getlast) - init - echo "$lastversion" - ;; - getfinal) - init - echo "$finalversion" - ;; - getcurrent) - init - get_current current - echo "$current" - ;; - get) - init - echo "Current final version: $finalversion" - echo "Last tagged version: $lastversion" - ;; - *) - echo "'$ACTION' is not a valid command, see --help for available commands." - ;; -esac diff --git a/versions.tf b/versions.tf index e9b6bc9562..602d086f47 100644 --- a/versions.tf +++ b/versions.tf @@ -2,10 +2,22 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.56.0" - local = ">= 1.4" - kubernetes = ">= 1.11.1" - cloudinit = ">= 2.0" + aws = { + source = "hashicorp/aws" + version = ">= 3.56" + } + local = { + source = "hashicorp/local" + version = ">= 1.4" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 1.11.1" + } + cloudinit = { + source = "hashicorp/cloudinit" + version = ">= 2.0" + } http = { source = "terraform-aws-modules/http" version = ">= 2.4.1" From e90c877a741ab3cc4215376a70f7bcc360b6a3d2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 22 Nov 2021 16:16:43 +0000 Subject: [PATCH 421/962] chore(release): version 17.24.0 [skip ci] # [17.24.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.23.0...v17.24.0) (2021-11-22) ### Bug Fixes * Added Deny for CreateLogGroup action in EKS cluster role ([#1594](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1594)) ([6959b9b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6959b9bae32309357bc97a85a1f09c7b590c8a6d)) * update CI/CD process to enable auto-release workflow ([#1698](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1698)) ([b876ff9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b876ff95136fbb419cbb33feaa8f354a053047e0)) ### Features * Add ability to define custom timeout for fargate profiles ([#1614](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1614)) ([b7539dc](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b7539dc220f6b5fe199d67569b6f3619ec00fdf0)) * Removed ng_depends_on variable and related hack ([#1672](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1672)) ([56e93d7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/56e93d77de58f311f1d1d7051f40bf77e7b03524)) --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2153e79f74..15937e80c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +# [17.24.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.23.0...v17.24.0) (2021-11-22) + + +### Bug Fixes + +* Added Deny for CreateLogGroup action in EKS cluster role ([#1594](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1594)) ([6959b9b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6959b9bae32309357bc97a85a1f09c7b590c8a6d)) +* update CI/CD process to enable auto-release workflow ([#1698](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1698)) ([b876ff9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b876ff95136fbb419cbb33feaa8f354a053047e0)) + + +### Features + +* Add ability to define custom timeout for fargate profiles ([#1614](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1614)) ([b7539dc](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b7539dc220f6b5fe199d67569b6f3619ec00fdf0)) +* Removed ng_depends_on variable and related hack ([#1672](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1672)) ([56e93d7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/56e93d77de58f311f1d1d7051f40bf77e7b03524)) + ## [v17.23.0] - 2021-11-02 FEATURES: From 3002be59ef25e67bc4dfc9a1d196174aeb871d8b Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Sun, 12 Dec 2021 21:49:57 +0100 Subject: [PATCH 422/962] chore: Added action to validate PR title (#1715) --- .github/semantic.yml | 30 ------------------- .github/workflows/pr-title.yml | 53 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 30 deletions(-) delete mode 100644 .github/semantic.yml create mode 100644 .github/workflows/pr-title.yml diff --git a/.github/semantic.yml b/.github/semantic.yml deleted file mode 100644 index 376c06a6a8..0000000000 --- a/.github/semantic.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Always validate the PR title, and ignore the commits -titleOnly: true - -# Always validate all commits, and ignore the PR title -commitsOnly: false - -# Always validate the PR title AND all the commits -titleAndCommits: false - -# Require at least one commit to be valid -# this is only relevant when using commitsOnly: true or titleAndCommits: true, -# which validate all commits by default -anyCommit: false - -# By default types specified in commitizen/conventional-commit-types is used. -# See: https://github.com/commitizen/conventional-commit-types/blob/v2.3.0/index.json -# You can override the valid types -types: - - feat - - fix - - improvement - - docs - - refactor - - test - - ci - - chore - -# Allow use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns") -# this is only relevant when using commitsOnly: true (or titleAndCommits: true) -allowMergeCommits: false diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml new file mode 100644 index 0000000000..7f5aec08ed --- /dev/null +++ b/.github/workflows/pr-title.yml @@ -0,0 +1,53 @@ +name: "Validate PR title" + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + steps: + # Please look up the latest version from + # https://github.com/amannn/action-semantic-pull-request/releases + - uses: amannn/action-semantic-pull-request@v3.4.6 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # Configure which types are allowed. + # Default: https://github.com/commitizen/conventional-commit-types + types: | + fix + feat + docs + ci + chore + BREAKING CHANGE + # Configure that a scope must always be provided. + requireScope: false + # Configure additional validation for the subject based on a regex. + # This example ensures the subject starts with an uppercase character. + subjectPattern: ^[A-Z].+$ + # If `subjectPattern` is configured, you can use this property to override + # the default error message that is shown when the pattern doesn't match. + # The variables `subject` and `title` can be used within the message. + subjectPatternError: | + The subject "{subject}" found in the pull request title "{title}" + didn't match the configured pattern. Please ensure that the subject + starts with an uppercase character. + # For work-in-progress PRs you can typically use draft pull requests + # from Github. However, private repositories on the free plan don't have + # this option and therefore this action allows you to opt-in to using the + # special "[WIP]" prefix to indicate this state. This will avoid the + # validation of the PR title and the pull request checks remain pending. + # Note that a second check will be reported if this is enabled. + wip: true + # When using "Squash and merge" on a PR with only one commit, GitHub + # will suggest using that commit message instead of the PR title for the + # merge commit, and it's easy to commit this by mistake. Enable this option + # to also validate the commit message for one commit PRs. + validateSingleCommit: false From 8d33a46170bca6af45348c93a20c72c4afa274ff Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Sun, 12 Dec 2021 21:55:51 +0100 Subject: [PATCH 423/962] chore: Updated allowed types for PR title (#1716) --- .github/workflows/pr-title.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 7f5aec08ed..4b16bf7a38 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -20,13 +20,13 @@ jobs: with: # Configure which types are allowed. # Default: https://github.com/commitizen/conventional-commit-types - types: | - fix - feat - docs - ci - chore - BREAKING CHANGE + types: + - fix + - feat + - docs + - ci + - chore + - BREAKING CHANGE # Configure that a scope must always be provided. requireScope: false # Configure additional validation for the subject based on a regex. From d569aa3554eded75289e9b4dcddd7e04d695a825 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Mon, 13 Dec 2021 15:26:21 +0100 Subject: [PATCH 424/962] chore: Fixed types in PR title (#1718) --- .github/workflows/pr-title.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 4b16bf7a38..b89795d729 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -20,13 +20,12 @@ jobs: with: # Configure which types are allowed. # Default: https://github.com/commitizen/conventional-commit-types - types: - - fix - - feat - - docs - - ci - - chore - - BREAKING CHANGE + types: | + fix + feat + docs + ci + chore # Configure that a scope must always be provided. requireScope: false # Configure additional validation for the subject based on a regex. From ee9f0c646a45ca9baa6174a036d1e09bcccb87b1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 5 Jan 2022 06:01:31 -0600 Subject: [PATCH 425/962] feat!: Removed support for launch configuration and replace `count` with `for_each` (#1680) --- .github/CONTRIBUTING.md | 1 - docs/upgrades.md => .github/UPGRADE-17.0.md | 0 .github/images/security_groups.svg | 1 + .github/images/user_data.svg | 1 + .github/workflows/release.yml | 1 + .github/workflows/stale-actions.yaml | 2 +- .pre-commit-config.yaml | 5 +- README.md | 919 +++++++++++++----- UPGRADE-18.0.md | 550 +++++++++++ aws_auth.tf | 92 -- data.tf | 104 -- docs/autoscaling.md | 98 -- docs/enable-docker-bridge-network.md | 23 - docs/faq.md | 235 ----- docs/iam-permissions.md | 155 --- docs/spot-instances.md | 114 --- examples/bottlerocket/README.md | 75 -- examples/bottlerocket/main.tf | 159 --- examples/bottlerocket/outputs.tf | 24 - examples/bottlerocket/userdata.toml | 24 - examples/bottlerocket/versions.tf | 26 - examples/complete/README.md | 75 +- examples/complete/main.tf | 401 ++++---- examples/complete/outputs.tf | 146 ++- examples/complete/versions.tf | 16 +- examples/eks_managed_node_group/README.md | 95 ++ examples/eks_managed_node_group/main.tf | 490 ++++++++++ examples/eks_managed_node_group/outputs.tf | 167 ++++ .../variables.tf | 0 examples/eks_managed_node_group/versions.tf | 14 + examples/fargate/README.md | 69 -- examples/fargate/main.tf | 235 ----- examples/fargate/outputs.tf | 24 - examples/fargate/versions.tf | 26 - examples/fargate_profile/README.md | 76 ++ examples/fargate_profile/main.tf | 164 ++++ examples/fargate_profile/outputs.tf | 167 ++++ .../{fargate => fargate_profile}/variables.tf | 0 .../fargate_profile}/versions.tf | 2 +- examples/instance_refresh/README.md | 82 -- examples/instance_refresh/main.tf | 306 ------ examples/instance_refresh/outputs.tf | 29 - examples/instance_refresh/versions.tf | 26 - examples/irsa/README.md | 70 -- examples/irsa/irsa.tf | 114 --- examples/irsa/main.tf | 115 --- examples/irsa/outputs.tf | 4 - examples/irsa/versions.tf | 26 - examples/irsa_autoscale_refresh/README.md | 102 ++ examples/irsa_autoscale_refresh/charts.tf | 294 ++++++ examples/irsa_autoscale_refresh/main.tf | 203 ++++ examples/irsa_autoscale_refresh/outputs.tf | 167 ++++ .../variables.tf | 0 examples/irsa_autoscale_refresh/versions.tf | 18 + examples/launch_templates/README.md | 65 -- examples/launch_templates/main.tf | 134 --- examples/launch_templates/outputs.tf | 19 - examples/launch_templates/pre_userdata.sh | 1 - examples/launch_templates/versions.tf | 22 - .../README.md | 70 -- .../disk_encryption_policy.tf | 78 -- .../launchtemplate.tf | 98 -- .../main.tf | 149 --- .../outputs.tf | 19 - .../templates/userdata.sh.tpl | 12 - .../variables.tf | 0 .../versions.tf | 22 - examples/managed_node_groups/README.md | 73 -- examples/managed_node_groups/main.tf | 148 --- examples/managed_node_groups/outputs.tf | 24 - examples/managed_node_groups/variables.tf | 48 - examples/managed_node_groups/versions.tf | 22 - examples/secrets_encryption/README.md | 66 -- examples/secrets_encryption/main.tf | 126 --- examples/secrets_encryption/outputs.tf | 19 - examples/secrets_encryption/variables.tf | 0 examples/secrets_encryption/versions.tf | 22 - examples/self_managed_node_group/README.md | 95 ++ examples/self_managed_node_group/main.tf | 392 ++++++++ examples/self_managed_node_group/outputs.tf | 167 ++++ .../variables.tf | 0 examples/self_managed_node_group/versions.tf | 18 + examples/user_data/README.md | 78 ++ examples/user_data/main.tf | 289 ++++++ examples/user_data/outputs.tf | 89 ++ .../templates/bottlerocket_custom.tpl | 7 + examples/user_data/templates/linux_custom.tpl | 10 + .../user_data/templates/windows_custom.tpl | 10 + .../variables.tf | 0 examples/user_data/versions.tf | 10 + fargate.tf | 16 - irsa.tf | 23 - kubectl.tf | 8 - locals.tf | 263 ----- main.tf | 319 +++--- modules/_user_data/README.md | 123 +++ modules/_user_data/main.tf | 78 ++ modules/_user_data/outputs.tf | 4 + modules/_user_data/variables.tf | 71 ++ modules/_user_data/versions.tf | 10 + modules/eks-managed-node-group/README.md | 175 ++++ modules/eks-managed-node-group/main.tf | 435 +++++++++ modules/eks-managed-node-group/outputs.tf | 75 ++ modules/eks-managed-node-group/variables.tf | 467 +++++++++ .../versions.tf | 2 +- modules/fargate-profile/README.md | 85 ++ modules/fargate-profile/main.tf | 80 ++ modules/fargate-profile/outputs.tf | 37 + modules/fargate-profile/variables.tf | 103 ++ modules/fargate-profile/versions.tf | 10 + modules/fargate/README.md | 73 -- modules/fargate/main.tf | 72 -- modules/fargate/outputs.tf | 29 - modules/fargate/variables.tf | 53 - modules/node_groups/README.md | 113 --- modules/node_groups/launch_template.tf | 146 --- modules/node_groups/locals.tf | 51 - modules/node_groups/main.tf | 105 -- modules/node_groups/outputs.tf | 14 - modules/node_groups/templates/userdata.sh.tpl | 34 - modules/node_groups/variables.tf | 71 -- modules/self-managed-node-group/README.md | 199 ++++ modules/self-managed-node-group/main.tf | 554 +++++++++++ modules/self-managed-node-group/outputs.tf | 147 +++ modules/self-managed-node-group/variables.tf | 579 +++++++++++ modules/self-managed-node-group/versions.tf | 14 + node_groups.tf | 410 +++++++- outputs.tf | 261 +++-- templates/aws_auth_cm.tpl | 37 + templates/bottlerocket_user_data.tpl | 7 + templates/kubeconfig.tpl | 38 - templates/linux_user_data.tpl | 14 + templates/userdata.sh.tpl | 10 - ...data_windows.tpl => windows_user_data.tpl} | 8 +- variables.tf | 488 ++++------ versions.tf | 20 +- workers.tf | 533 ---------- workers_launch_template.tf | 647 ------------ 138 files changed, 8986 insertions(+), 6859 deletions(-) rename docs/upgrades.md => .github/UPGRADE-17.0.md (100%) create mode 100644 .github/images/security_groups.svg create mode 100644 .github/images/user_data.svg create mode 100644 UPGRADE-18.0.md delete mode 100644 aws_auth.tf delete mode 100644 data.tf delete mode 100644 docs/autoscaling.md delete mode 100644 docs/enable-docker-bridge-network.md delete mode 100644 docs/faq.md delete mode 100644 docs/iam-permissions.md delete mode 100644 docs/spot-instances.md delete mode 100644 examples/bottlerocket/README.md delete mode 100644 examples/bottlerocket/main.tf delete mode 100644 examples/bottlerocket/outputs.tf delete mode 100644 examples/bottlerocket/userdata.toml delete mode 100644 examples/bottlerocket/versions.tf create mode 100644 examples/eks_managed_node_group/README.md create mode 100644 examples/eks_managed_node_group/main.tf create mode 100644 examples/eks_managed_node_group/outputs.tf rename examples/{bottlerocket => eks_managed_node_group}/variables.tf (100%) create mode 100644 examples/eks_managed_node_group/versions.tf delete mode 100644 examples/fargate/README.md delete mode 100644 examples/fargate/main.tf delete mode 100644 examples/fargate/outputs.tf delete mode 100644 examples/fargate/versions.tf create mode 100644 examples/fargate_profile/README.md create mode 100644 examples/fargate_profile/main.tf create mode 100644 examples/fargate_profile/outputs.tf rename examples/{fargate => fargate_profile}/variables.tf (100%) rename {modules/fargate => examples/fargate_profile}/versions.tf (82%) delete mode 100644 examples/instance_refresh/README.md delete mode 100644 examples/instance_refresh/main.tf delete mode 100644 examples/instance_refresh/outputs.tf delete mode 100644 examples/instance_refresh/versions.tf delete mode 100644 examples/irsa/README.md delete mode 100644 examples/irsa/irsa.tf delete mode 100644 examples/irsa/main.tf delete mode 100644 examples/irsa/outputs.tf delete mode 100644 examples/irsa/versions.tf create mode 100644 examples/irsa_autoscale_refresh/README.md create mode 100644 examples/irsa_autoscale_refresh/charts.tf create mode 100644 examples/irsa_autoscale_refresh/main.tf create mode 100644 examples/irsa_autoscale_refresh/outputs.tf rename examples/{instance_refresh => irsa_autoscale_refresh}/variables.tf (100%) create mode 100644 examples/irsa_autoscale_refresh/versions.tf delete mode 100644 examples/launch_templates/README.md delete mode 100644 examples/launch_templates/main.tf delete mode 100644 examples/launch_templates/outputs.tf delete mode 100644 examples/launch_templates/pre_userdata.sh delete mode 100644 examples/launch_templates/versions.tf delete mode 100644 examples/launch_templates_with_managed_node_groups/README.md delete mode 100644 examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf delete mode 100644 examples/launch_templates_with_managed_node_groups/launchtemplate.tf delete mode 100644 examples/launch_templates_with_managed_node_groups/main.tf delete mode 100644 examples/launch_templates_with_managed_node_groups/outputs.tf delete mode 100644 examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl delete mode 100644 examples/launch_templates_with_managed_node_groups/variables.tf delete mode 100644 examples/launch_templates_with_managed_node_groups/versions.tf delete mode 100644 examples/managed_node_groups/README.md delete mode 100644 examples/managed_node_groups/main.tf delete mode 100644 examples/managed_node_groups/outputs.tf delete mode 100644 examples/managed_node_groups/variables.tf delete mode 100644 examples/managed_node_groups/versions.tf delete mode 100644 examples/secrets_encryption/README.md delete mode 100644 examples/secrets_encryption/main.tf delete mode 100644 examples/secrets_encryption/outputs.tf delete mode 100644 examples/secrets_encryption/variables.tf delete mode 100644 examples/secrets_encryption/versions.tf create mode 100644 examples/self_managed_node_group/README.md create mode 100644 examples/self_managed_node_group/main.tf create mode 100644 examples/self_managed_node_group/outputs.tf rename examples/{irsa => self_managed_node_group}/variables.tf (100%) create mode 100644 examples/self_managed_node_group/versions.tf create mode 100644 examples/user_data/README.md create mode 100644 examples/user_data/main.tf create mode 100644 examples/user_data/outputs.tf create mode 100644 examples/user_data/templates/bottlerocket_custom.tpl create mode 100644 examples/user_data/templates/linux_custom.tpl create mode 100644 examples/user_data/templates/windows_custom.tpl rename examples/{launch_templates => user_data}/variables.tf (100%) create mode 100644 examples/user_data/versions.tf delete mode 100644 fargate.tf delete mode 100644 irsa.tf delete mode 100644 kubectl.tf delete mode 100644 locals.tf create mode 100644 modules/_user_data/README.md create mode 100644 modules/_user_data/main.tf create mode 100644 modules/_user_data/outputs.tf create mode 100644 modules/_user_data/variables.tf create mode 100644 modules/_user_data/versions.tf create mode 100644 modules/eks-managed-node-group/README.md create mode 100644 modules/eks-managed-node-group/main.tf create mode 100644 modules/eks-managed-node-group/outputs.tf create mode 100644 modules/eks-managed-node-group/variables.tf rename modules/{node_groups => eks-managed-node-group}/versions.tf (89%) create mode 100644 modules/fargate-profile/README.md create mode 100644 modules/fargate-profile/main.tf create mode 100644 modules/fargate-profile/outputs.tf create mode 100644 modules/fargate-profile/variables.tf create mode 100644 modules/fargate-profile/versions.tf delete mode 100644 modules/fargate/README.md delete mode 100644 modules/fargate/main.tf delete mode 100644 modules/fargate/outputs.tf delete mode 100644 modules/fargate/variables.tf delete mode 100644 modules/node_groups/README.md delete mode 100644 modules/node_groups/launch_template.tf delete mode 100644 modules/node_groups/locals.tf delete mode 100644 modules/node_groups/main.tf delete mode 100644 modules/node_groups/outputs.tf delete mode 100644 modules/node_groups/templates/userdata.sh.tpl delete mode 100644 modules/node_groups/variables.tf create mode 100644 modules/self-managed-node-group/README.md create mode 100644 modules/self-managed-node-group/main.tf create mode 100644 modules/self-managed-node-group/outputs.tf create mode 100644 modules/self-managed-node-group/variables.tf create mode 100644 modules/self-managed-node-group/versions.tf create mode 100644 templates/aws_auth_cm.tpl create mode 100644 templates/bottlerocket_user_data.tpl delete mode 100644 templates/kubeconfig.tpl create mode 100644 templates/linux_user_data.tpl delete mode 100644 templates/userdata.sh.tpl rename templates/{userdata_windows.tpl => windows_user_data.tpl} (53%) delete mode 100644 workers.tf delete mode 100644 workers_launch_template.tf diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 65df33743e..53127506fa 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -31,4 +31,3 @@ To generate changelog, Pull Requests or Commits must have semantic and must foll - `chore:` for chores stuff The `chore` prefix skipped during changelog generation. It can be used for `chore: update changelog` commit message by example. - diff --git a/docs/upgrades.md b/.github/UPGRADE-17.0.md similarity index 100% rename from docs/upgrades.md rename to .github/UPGRADE-17.0.md diff --git a/.github/images/security_groups.svg b/.github/images/security_groups.svg new file mode 100644 index 0000000000..6b120e98ba --- /dev/null +++ b/.github/images/security_groups.svg @@ -0,0 +1 @@ + diff --git a/.github/images/user_data.svg b/.github/images/user_data.svg new file mode 100644 index 0000000000..a36b8051e4 --- /dev/null +++ b/.github/images/user_data.svg @@ -0,0 +1 @@ + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 141937d863..63efd61394 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ on: - main - master paths: + - '**/*.tpl' - '**/*.py' - '**/*.tf' diff --git a/.github/workflows/stale-actions.yaml b/.github/workflows/stale-actions.yaml index 93c90dfcd1..d2a5f6b96e 100644 --- a/.github/workflows/stale-actions.yaml +++ b/.github/workflows/stale-actions.yaml @@ -29,4 +29,4 @@ jobs: days-before-close: 10 delete-branch: true close-issue-message: This issue was automatically closed because of stale in 10 days - close-pr-message: This PR was automatically closed because of stale in 10 days \ No newline at end of file + close-pr-message: This PR was automatically closed because of stale in 10 days diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 20e651ab6a..006410918f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.58.0 + rev: v1.62.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -17,7 +17,7 @@ repos: - '--args=--only=terraform_documented_variables' - '--args=--only=terraform_typed_variables' - '--args=--only=terraform_module_pinned_source' - # - '--args=--only=terraform_naming_convention' + - '--args=--only=terraform_naming_convention' - '--args=--only=terraform_required_version' - '--args=--only=terraform_required_providers' - '--args=--only=terraform_standard_module_structure' @@ -26,3 +26,4 @@ repos: rev: v4.0.1 hooks: - id: check-merge-conflict + - id: end-of-file-fixer diff --git a/README.md b/README.md index 26c6f11cec..c93a37a8d8 100644 --- a/README.md +++ b/README.md @@ -1,114 +1,641 @@ # AWS EKS Terraform module -[![Lint Status](https://github.com/terraform-aws-modules/terraform-aws-eks/workflows/Lint/badge.svg)](https://github.com/terraform-aws-modules/terraform-aws-eks/actions) -[![LICENSE](https://img.shields.io/github/license/terraform-aws-modules/terraform-aws-eks)](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/LICENSE) +Terraform module which creates AWS EKS (Kubernetes) resources + +## Available Features + +- AWS EKS Cluster +- AWS EKS Cluster Addons +- AWS EKS Identity Provider Configuration +- All [node types](https://docs.aws.amazon.com/eks/latest/userguide/eks-compute.html) are supported: + - [EKS Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) + - [Self Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) + - [Fargate Profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) +- Support for custom AMI, custom launch template, and custom user data +- Support for Amazon Linux 2 EKS Optimized AMI and Bottlerocket nodes + - Windows based node support is limited to a default user data template that is provided due to the lack of Windows support and manual steps required to provision Windows based EKS nodes +- Support for module created security group, bring your own security groups, as well as adding additional security group rules to the module created security group(s) +- Support for providing maps of node groups/Fargate profiles to the cluster module definition or use separate node group/Fargate profile sub-modules +- Provisions to provide node group/Fargate profile "default" settings - useful for when creating multiple node groups/Fargate profiles where you want to set a common set of configurations once, and then individual control only select features + +## Usage -Terraform module which creates Kubernetes cluster resources on AWS EKS. +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" -## Features + cluster_name = "my-cluster" + cluster_version = "1.21" + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + cluster_addons = { + coredns = { + resolve_conflicts = "OVERWRITE" + } + kube-proxy = {} + vpc-cni = { + resolve_conflicts = "OVERWRITE" + } + } + + cluster_encryption_config = [{ + provider_key_arn = "ac01234b-00d9-40f6-ac95-e42345f78b00" + resources = ["secrets"] + }] + + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + + # Self Managed Node Group(s) + self_managed_node_group_defaults = { + instance_type = "m6i.large" + update_launch_template_default_version = true + iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"] + } + + self_managed_node_groups = { + one = { + name = "spot-1" + + public_ip = true + max_size = 5 + desired_size = 2 + + use_mixed_instances_policy = true + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 10 + spot_allocation_strategy = "capacity-optimized" + } + + override = [ + { + instance_type = "m5.large" + weighted_capacity = "1" + }, + { + instance_type = "m6i.large" + weighted_capacity = "2" + }, + ] + } + + pre_bootstrap_user_data = <<-EOT + echo "foo" + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + post_bootstrap_user_data = <<-EOT + cd /tmp + sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm + sudo systemctl enable amazon-ssm-agent + sudo systemctl start amazon-ssm-agent + EOT + } + } + + # EKS Managed Node Group(s) + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] + vpc_security_group_ids = [aws_security_group.additional.id] + } + + eks_managed_node_groups = { + blue = {} + green = { + min_size = 1 + max_size = 10 + desired_size = 1 + + instance_types = ["t3.large"] + capacity_type = "SPOT" + labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + taints = { + dedicated = { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + } + tags = { + ExtraTag = "example" + } + } + } + + # Fargate Profile(s) + fargate_profiles = { + default = { + name = "default" + selectors = [ + { + namespace = "kube-system" + labels = { + k8s-app = "kube-dns" + } + }, + { + namespace = "default" + } + ] + + tags = { + Owner = "test" + } + + timeouts = { + create = "20m" + delete = "20m" + } + } + } -- Create an EKS cluster -- All node types are supported: - - [Managed Node Groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) - - [Self-managed Nodes](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) - - [Fargate](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) -- Support AWS EKS Optimized or Custom AMI -- Create or manage security groups that allow communication and coordination + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` -## Important note +## Node Group Configuration -Kubernetes is evolving a lot, and each minor version includes new features, fixes, or changes. +⚠️ The configurations shown below are referenced from within the root EKS module; there will be slight differences in the default values provided when compared to the underlying sub-modules (`eks-managed-node-group`, `self-managed-node-group`, and `fargate-profile`). -**Always check [Kubernetes Release Notes](https://kubernetes.io/docs/setup/release/notes/) before updating the major version, and [CHANGELOG.md](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/CHANGELOG.md) for all changes in this EKS module.** +### EKS Managed Node Groups -You also need to ensure that your applications and add ons are updated, or workloads could fail after the upgrade is complete. For action, you may need to take before upgrading, see the steps in the [EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). +ℹ️ Only the pertinent attributes are shown for brevity -## Usage example +1. AWS EKS Managed Node Group can provide its own launch template and utilize the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version: ```hcl -data "aws_eks_cluster" "eks" { - name = module.eks.cluster_id -} + eks_managed_node_groups = { + default = {} + } +``` -data "aws_eks_cluster_auth" "eks" { - name = module.eks.cluster_id -} +2. AWS EKS Managed Node Group also offers native, default support for Bottlerocket OS by simply specifying the AMI type: -provider "kubernetes" { - host = data.aws_eks_cluster.eks.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.eks.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.eks.token -} +```hcl + eks_managed_node_groups = { + bottlerocket_default = { + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + } + } +``` -module "eks" { - source = "terraform-aws-modules/eks/aws" +3. AWS EKS Managed Node Groups allow you to extend configurations by providing your own launch template and user data that is merged with what the service provides. For example, to provide additional user data before the nodes are bootstrapped as well as supply additional arguments to the bootstrap script: + +```hcl + eks_managed_node_groups = { + extend_config = { + # This is supplied to the AWS EKS Optimized AMI + # bootstrap script https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh + bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" + + # This user data will be injected prior to the user data provided by the + # AWS EKS Managed Node Group service (contains the actually bootstrap configuration) + pre_bootstrap_user_data = <<-EOT + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + EOT + } + } +``` + +4. The same configurations extension is offered when utilizing Bottlerocket OS AMIs, but the user data is slightly different. Bottlerocket OS uses a TOML user data file and you can provide additional configuration settings via the `bootstrap_extra_args` variable which gets merged into what is provided by the AWS EKS Managed Node Service: + +```hcl + eks_managed_node_groups = { + bottlerocket_extend_config = { + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + + # this will get added to what AWS provides + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + EOT + } + } +``` + +5. Users can also utilize a custom AMI, but doing so means that AWS EKS Managed Node Group will NOT inject the necessary bootstrap script and configurations into the user data supplied to the launch template. When using a custom AMI, users must also opt in to bootstrapping the nodes via user data and either use the module default user data template or provide your own user data template file: +```hcl + eks_managed_node_groups = { + custom_ami = { + ami_id = "ami-0caf35bc73450c396" + + # By default, EKS managed node groups will not append bootstrap script; + # this adds it back in using the default template provided by the module + # Note: this assumes the AMI provided is an EKS optimized AMI derivative + enable_bootstrap_user_data = true + + bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" + + pre_bootstrap_user_data = <<-EOT + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + EOT + + # Because we have full control over the user data supplied, we can also run additional + # scripts/configuration changes after the bootstrap script has been run + post_bootstrap_user_data = <<-EOT + echo "you are free little kubelet!" + EOT + } + } +``` + +6. Similarly, for Bottlerocket there is similar support: + +```hcl + eks_managed_node_groups = { + bottlerocket_custom_ami = { + ami_id = "ami-0ff61e0bcfc81dc94" + platform = "bottlerocket" + + # use module user data template to bootstrap + enable_bootstrap_user_data = true + # this will get added to the template + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + + [settings.kubernetes.node-labels] + "label1" = "foo" + "label2" = "bar" + + [settings.kubernetes.node-taints] + "dedicated" = "experimental:PreferNoSchedule" + "special" = "true:NoSchedule" + EOT + } + } +``` + +See the [`examples/eks_managed_node_group/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group) for a working example of these configurations. + +### Self Managed Node Groups + +ℹ️ Only the pertinent attributes are shown for brevity + +1. By default, the `self-managed-node-group` sub-module will use the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version: + +```hcl cluster_version = "1.21" - cluster_name = "my-cluster" - vpc_id = "vpc-1234556abcdef" - subnets = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] - - worker_groups = [ - { - instance_type = "m4.large" - asg_max_size = 5 + + # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.21 + self_managed_node_groups = { + default = {} + } +``` + +2. To use Bottlerocket, specify the `platform` as `bottlerocket` and supply the Bottlerocket AMI. The module provided user data for Bottlerocket will be used to bootstrap the nodes created: + +```hcl + cluster_version = "1.21" + + self_managed_node_groups = { + bottlerocket = { + platform = "bottlerocket" + ami_id = data.aws_ami.bottlerocket_ami.id } - ] -} + } +``` + +### Fargate Profiles + +Fargate profiles are rather straightforward. Simply supply the necessary information for the desired profile(s). See the [`examples/fargate_profile/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile) for a working example of the various configurations. + +### Mixed Node Groups + +ℹ️ Only the pertinent attributes are shown for brevity + +Users are free to mix and match the different node group types that meet their needs. For example, the following are just an example of the different possibilities: +- AWS EKS Cluster with one or more AWS EKS Managed Node Groups +- AWS EKS Cluster with one or more Self Managed Node Groups +- AWS EKS Cluster with one or more Fargate profiles +- AWS EKS Cluster with one or more AWS EKS Managed Node Groups, one or more Self Managed Node Groups, one or more Fargate profiles + +It is also possible to configure the various node groups of each family differently. Node groups may also be defined outside of the root `eks` module definition by using the provided sub-modules. There are no restrictions on the the various different possibilities provided by the module. + +```hcl + self_managed_node_group_defaults = { + vpc_security_group_ids = [aws_security_group.additional.id] + iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"] + } + + self_managed_node_groups = { + one = { + name = "spot-1" + + public_ip = true + max_size = 5 + desired_size = 2 + + use_mixed_instances_policy = true + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 10 + spot_allocation_strategy = "capacity-optimized" + } + + override = [ + { + instance_type = "m5.large" + weighted_capacity = "1" + }, + { + instance_type = "m6i.large" + weighted_capacity = "2" + }, + ] + } + + pre_bootstrap_user_data = <<-EOT + echo "foo" + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + post_bootstrap_user_data = <<-EOT + cd /tmp + sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm + sudo systemctl enable amazon-ssm-agent + sudo systemctl start amazon-ssm-agent + EOT + } + } + + # EKS Managed Node Group(s) + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] + vpc_security_group_ids = [aws_security_group.additional.id] + } + + eks_managed_node_groups = { + blue = {} + green = { + min_size = 1 + max_size = 10 + desired_size = 1 + + instance_types = ["t3.large"] + capacity_type = "SPOT" + labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + taints = { + dedicated = { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + } + + update_config = { + max_unavailable_percentage = 50 # or set `max_unavailable` + } + + tags = { + ExtraTag = "example" + } + } + } + + # Fargate Profile(s) + fargate_profiles = { + default = { + name = "default" + selectors = [ + { + namespace = "kube-system" + labels = { + k8s-app = "kube-dns" + } + }, + { + namespace = "default" + } + ] + + tags = { + Owner = "test" + } + + timeouts = { + create = "20m" + delete = "20m" + } + } + } +``` + +See the [`examples/complete/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete) for a working example of these configurations. + +### Default configurations + +Each node group type (EKS managed node group, self managed node group, or Fargate profile) provides a default configuration setting that allows users to provide their own default configuration instead of the module's default configuration. This allows users to set a common set of defaults for their node groups and still maintain the ability to override these settings within the specific node group definition. The order of precedence for each node group type roughly follows (from highest to least precedence): +- Node group individual configuration + - Node group family default configuration + - Module default configuration + +These are provided via the following variables for the respective node group family: +- `eks_managed_node_group_defaults` +- `self_managed_node_group_defaults` +- `fargate_profile_defaults` + +For example, the following creates 4 AWS EKS Managed Node Groups: + +```hcl + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] + } + + eks_managed_node_groups = { + # Uses defaults provided by module with the default settings above overriding the module defaults + default = {} + + # This further overrides the instance types used + compute = { + instance_types = ["c5.large", "c6i.large", "c6d.large"] + } + + # This further overrides the instance types and disk size used + persistent = { + disk_size = 1024 + instance_types = ["r5.xlarge", "r6i.xlarge", "r5b.xlarge"] + } + + # This overrides the OS used + bottlerocket = { + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + } + } ``` -There is also a [complete example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete) which shows large set of features available in the module. +## Module Design Considerations + +### General Notes + +While the module is designed to be flexible and support as many use cases and configurations as possible, there is a limit to what first class support can be provided without over-burdening the complexity of the module. Below are a list of general notes on the design intent captured by this module which hopefully explains some of the decisions that are, or will be made, in terms of what is added/supported natively by the module: + +- Despite the addition of Windows Subsystem for Linux (WSL for short), containerization technology is very much a suite of Linux constructs and therefore Linux is the primary OS supported by this module. In addition, due to the first class support provided by AWS, Bottlerocket OS and Fargate Profiles are also very much fully supported by this module. This module does not make any attempt to NOT support Windows, as in preventing the usage of Windows based nodes, however it is up to users to put in additional effort in order to operate Windows based nodes when using the module. User can refer to the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html) for further details. What this means is: + - AWS EKS Managed Node Groups default to `linux` as the `platform`, but `bottlerocket` is also supported by AWS (`windows` is not supported by AWS EKS Managed Node groups) + - AWS Self Managed Node Groups also default to `linux` and the default AMI used is the latest AMI for the selected Kubernetes version. If you wish to use a different OS or AMI then you will need to opt in to the necessary configurations to ensure the correct AMI is used in conjunction with the necessary user data to ensure the nodes are launched and joined to your cluster successfully. +- AWS EKS Managed Node groups are currently the preferred route over Self Managed Node Groups for compute nodes. Both operate very similarly - both are backed by autoscaling groups and launch templates deployed and visible within your account. However, AWS EKS Managed Node groups provide a better user experience and offer a more "managed service" experience and therefore has precedence over Self Managed Node Groups. That said, there are currently inherent limitations as AWS continues to rollout additional feature support similar to the level of customization you can achieve with Self Managed Node Groups. When requesting added feature support for AWS EKS Managed Node groups, please ensure you have verified that the feature(s) are 1) supported by AWS and 2) supported by the Terraform AWS provider before submitting a feature request. +- Due to the plethora of tooling and different manners of configuring your cluster, cluster configuration is intentionally left out of the module in order to simplify the module for a broader user base. Previous module versions provided support for managing the aws-auth configmap via the Kubernetes Terraform provider using the now deprecated aws-iam-authenticator; these are no longer included in the module. This module strictly focuses on the infrastructure resources to provision an EKS cluster as well as any supporting AWS resources. How the internals of the cluster are configured and managed is up to users and is outside the scope of this module. There is an output attribute, `aws_auth_configmap_yaml`, that has been provided that can be useful to help bridge this transition. Please see the various examples provided where this attribute is used to ensure that self managed node groups or external node groups have their IAM roles appropriately mapped to the aws-auth configmap. How users elect to manage the aws-auth configmap is left up to their choosing. + +### User Data & Bootstrapping + +There are a multitude of different possible configurations for how module users require their user data to be configured. In order to better support the various combinations from simple, out of the box support provided by the module to full customization of the user data using a template provided by users - the user data has been abstracted out to its own module. Users can see the various methods of using and providing user data through the [user data examples](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) as well more detailed information on the design and possible configurations via the [user data module itself](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/_user_data) + +In general (tl;dr): +- AWS EKS Managed Node Groups + - `linux` platform (default) -> user data is pre-pended to the AWS provided bootstrap user data (bash/shell script) when using the AWS EKS provided AMI, otherwise users need to opt in via `enable_bootstrap_user_data` and use the module provided user data template or provide their own user data template to bootstrap nodes to join the cluster + - `bottlerocket` platform -> user data is merged with the AWS provided bootstrap user data (TOML file) when using the AWS EKS provided AMI, otherwise users need to opt in via `enable_bootstrap_user_data` and use the module provided user data template or provide their own user data template to bootstrap nodes to join the cluster +- Self Managed Node Groups + - `linux` platform (default) -> the user data template (bash/shell script) provided by the module is used as the default; users are able to provide their own user data template + - `bottlerocket` platform -> the user data template (TOML file) provided by the module is used as the default; users are able to provide their own user data template + - `windows` platform -> the user data template (powershell/PS1 script) provided by the module is used as the default; users are able to provide their own user data template + +Module provided default templates can be found under the [templates directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/templates) + +### Security Groups + +- Cluster Security Group + - This module by default creates a cluster security group ("additional" security group when viewed from the console) in addition to the default security group created by the AWS EKS service. This "additional" security group allows users to customize inbound and outbound rules via the module as they see fit + - The default inbound/outbound rules provided by the module are derived from the [AWS minimum recommendations](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) in addition to NTP and HTTPS public internet egress rules (without, these show up in VPC flow logs as rejects - they are used for clock sync and downloading necessary packages/updates) + - The minimum inbound/outbound rules are provided for cluster and node creation to succeed without errors, but users will most likely need to add the necessary port and protocol for node-to-node communication (this is user specific based on how nodes are configured to communicate across the cluster) + - Users have the ability to opt out of the security group creation and instead provide their own externally created security group if so desired + - The security group that is created is designed to handle the bare minimum communication necessary between the control plane and the nodes, as well as any external egress to allow the cluster to successfully launch without error + - Users also have the option to supply additional, externally created security groups to the cluster as well via the `cluster_additional_security_group_ids` variable -## Submodules +- Node Group Security Group(s) + - Each node group (EKS Managed Node Group and Self Managed Node Group) by default creates its own security group. By default, this security group does not contain any additional security group rules. It is merely an "empty container" that offers users the ability to opt into any addition inbound our outbound rules as necessary + - Users also have the option to supply their own, and/or additional, externally created security group(s) to the node group as well via the `vpc_security_group_ids` variable -Root module calls these modules which can also be used separately to create independent resources: +The security groups created by this module are depicted in the image shown below along with their default inbound/outbound rules: -- [fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/fargate) - creates Fargate profiles, see [examples/fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate) for detailed examples. - +

+ Security Groups +

## Notes -- By default, this module manages the `aws-auth` configmap for you (`manage_aws_auth=true`). To avoid the following [issue](https://github.com/aws/containers-roadmap/issues/654) where the EKS creation is `ACTIVE` but not ready. We implemented a "retry" logic with a [fork of the http provider](https://github.com/terraform-aws-modules/terraform-provider-http). This fork adds the support of a self-signed CA certificate. The original PR can be found [here](https://github.com/hashicorp/terraform-provider-http/pull/29). +- Setting `instance_refresh_enabled = true` will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. See the [instance_refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/instance_refresh) example provided. -- Setting `instance_refresh_enabled = true` will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. Find the complete example here [instance_refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/instance_refresh). +
Frequently Asked Questions
-## Documentation +

Why are nodes not being registered?

-### Official docs +Often an issue caused by one of two reasons: +1. Networking or endpoint mis-configuration. +2. Permissions (IAM/RBAC) -- [Amazon Elastic Kubernetes Service (Amazon EKS)](https://docs.aws.amazon.com/eks/latest/userguide/). +At least one of the cluster public or private endpoints must be enabled to access the cluster to work. If you require a public endpoint, setting up both (public and private) and restricting the public endpoint via setting `cluster_endpoint_public_access_cidrs` is recommended. More info regarding communication with an endpoint is available [here](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html). -### Module docs +Nodes need to be able to contact the EKS cluster endpoint. By default, the module only creates a public endpoint. To access the endpoint, the nodes need outgoing internet access: -- [Autoscaling](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/autoscaling.md): How to enable worker node autoscaling. -- [Enable Docker Bridge Network](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/enable-docker-bridge-network.md): How to enable the docker bridge network when using the EKS-optimized AMI, which disables it by default. -- [Spot instances](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/spot-instances.md): How to use spot instances with this module. -- [IAM Permissions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md): Minimum IAM permissions needed to setup EKS Cluster. -- [FAQ](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md): Frequently Asked Questions +- Nodes in private subnets: via a NAT gateway or instance along with the appropriate routing rules +- Nodes in public subnets: ensure that nodes are launched with public IPs is enabled (either through the module here or your subnet setting defaults) -## Examples +Important: If you apply only the public endpoint and configure the `cluster_endpoint_public_access_cidrs` to restrict access, know that EKS nodes will also use the public endpoint and you must allow access to the endpoint. If not, then your nodes will fail to work correctly. -There are detailed examples available for you to see how certain features of this module can be used in a straightforward way. Make sure to check them and run them before opening an issue. [Here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/iam-permissions.md) you can find the list of the minimum IAM Permissions required to create EKS cluster. +Cluster private endpoint can also be enabled by setting `cluster_endpoint_private_access = true` on this module. Node communication to the endpoint stays within the VPC. Ensure that VPC DNS resolution and hostnames are also enabled for your VPC when the private endpoint is enabled. -- [Complete](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete) - Create EKS Cluster with all available workers types in various combinations with many of supported features. -- [Bottlerocket](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/bottlerocket) - Create EKS cluster using [Bottlerocket AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami-bottlerocket.html). -- [Fargate](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate) - Create EKS cluster with [Fargate profiles](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) and attach Fargate profiles to an existing EKS cluster. - -## Contributing +Nodes need to be able to connect to other AWS services plus pull down container images from container registries (ECR). If for some reason you cannot enable public internet access for nodes you can add VPC endpoints to the relevant services: EC2 API, ECR API, ECR DKR and S3. -Report issues/questions/feature requests on in the [issues](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/new) section. +

How can I work with the cluster if I disable the public endpoint?

-Full contributing [guidelines are covered here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/.github/CONTRIBUTING.md). +You have to interact with the cluster from within the VPC that it is associated with; either through a VPN connection, a bastion EC2 instance, etc. -## Authors +

How can I stop Terraform from removing the EKS tags from my VPC and subnets?

-This module has been originally created by [Brandon O'Connor](https://github.com/brandoconnor), and was maintained by [Max Williams](https://github.com/max-rocket-internet), [Thierno IB. BARRY](https://github.com/barryib) and many more [contributors listed here](https://github.com/terraform-aws-modules/terraform-aws-eks/graphs/contributors)! +You need to add the tags to the Terraform definition of the VPC and subnets yourself. See the [basic example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). -## License +An alternative is to use the aws provider's [`ignore_tags` variable](https://www.terraform.io/docs/providers/aws/#ignore_tags-configuration-block). However this can also cause terraform to display a perpetual difference. + +

Why are there no changes when a node group's desired count is modified?

+ +The module is configured to ignore this value. Unfortunately, Terraform does not support variables within the `lifecycle` block. The setting is ignored to allow the cluster autoscaler to work correctly so that `terraform apply` does not accidentally remove running workers. You can change the desired count via the CLI or console if you're not using the cluster autoscaler. + +If you are not using autoscaling and want to control the number of nodes via terraform, set the `min_size` and `max_size` for node groups. Before changing those values, you must satisfy AWS `desired_size` constraints (which must be between new min/max values). + +

Why are nodes not recreated when the `launch_template` is recreated?

+ +By default the ASG for a self-managed node group is not configured to be recreated when the launch configuration or template changes; you will need to use a process to drain and cycle the nodes. + +If you are NOT using the cluster autoscaler: + +- Add a new instance +- Drain an old node `kubectl drain --force --ignore-daemonsets --delete-local-data ip-xxxxxxx.eu-west-1.compute.internal` +- Wait for pods to be Running +- Terminate the old node instance. ASG will start a new instance +- Repeat the drain and delete process until all old nodes are replaced + +If you are using the cluster autoscaler: + +- Drain an old node `kubectl drain --force --ignore-daemonsets --delete-local-data ip-xxxxxxx.eu-west-1.compute.internal` +- Wait for pods to be Running +- Cluster autoscaler will create new nodes when required +- Repeat until all old nodes are drained +- Cluster autoscaler will terminate the old nodes after 10-60 minutes automatically + +You can also use a third-party tool like Gruntwork's kubergrunt. See the [`eks deploy`](https://github.com/gruntwork-io/kubergrunt#deploy) subcommand. + +Alternatively, use a managed node group instead. +

How can I use Windows workers?

-Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. +To enable Windows support for your EKS cluster, you should apply some configuration manually. See the [Enabling Windows Support (Windows/MacOS/Linux)](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support). + +Windows based nodes require an additional cluster role (`eks:kube-proxy-windows`). + +

Worker nodes with labels do not join a 1.16+ cluster

+ +As of Kubernetes 1.16, kubelet restricts which labels with names in the `kubernetes.io` namespace can be applied to nodes. Labels such as `kubernetes.io/lifecycle=spot` are no longer allowed; instead use `node.kubernetes.io/lifecycle=spot` + +See your Kubernetes version's documentation for the `--node-labels` kubelet flag for the allowed prefixes. [Documentation for 1.16](https://v1-16.docs.kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) + +
+ +## Examples + +- [Complete](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete): EKS Cluster using all available node group types in various combinations demonstrating many of the supported features and configurations +- [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group): EKS Cluster using EKS managed node groups +- [Fargate Profile](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile): EKS cluster using [Fargate Profiles](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) +- [IRSA, Node Autoscaler, Instance Refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa_autoscale_instance_refresh): EKS Cluster using self-managed node group demonstrating how to enable/utilize instance refresh configuration along with node termination handler +- [Self Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self_managed_node_group): EKS Cluster using self-managed node groups +- [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self_managed_node_group): Various supported methods of providing necessary bootstrap scripts and configuration settings via user data + +## Contributing + +Report issues/questions/feature requests via [issues](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/new) +Full contributing [guidelines are covered here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/.github/CONTRIBUTING.md) ## Requirements @@ -116,195 +643,129 @@ Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraf | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [cloudinit](#requirement\_cloudinit) | >= 2.0 | -| [http](#requirement\_http) | >= 2.4.1 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | -| [local](#requirement\_local) | >= 1.4 | +| [aws](#requirement\_aws) | >= 3.64 | +| [tls](#requirement\_tls) | >= 2.2 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [http](#provider\_http) | >= 2.4.1 | -| [kubernetes](#provider\_kubernetes) | >= 1.11.1 | -| [local](#provider\_local) | >= 1.4 | +| [aws](#provider\_aws) | >= 3.64 | +| [tls](#provider\_tls) | >= 2.2 | ## Modules | Name | Source | Version | |------|--------|---------| -| [fargate](#module\_fargate) | ./modules/fargate | n/a | -| [node\_groups](#module\_node\_groups) | ./modules/node_groups | n/a | +| [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ./modules/eks-managed-node-group | n/a | +| [fargate\_profile](#module\_fargate\_profile) | ./modules/fargate-profile | n/a | +| [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ./modules/self-managed-node-group | n/a | ## Resources | Name | Type | |------|------| -| [aws_autoscaling_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | -| [aws_autoscaling_group.workers_launch_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | | [aws_cloudwatch_log_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_eks_addon.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | -| [aws_iam_instance_profile.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | -| [aws_iam_instance_profile.workers_launch_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_eks_identity_provider_config.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_identity_provider_config) | resource | | [aws_iam_openid_connect_provider.oidc_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource | -| [aws_iam_policy.cluster_deny_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cluster_elb_sl_role_creation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cluster_AmazonEKSVPCResourceControllerPolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cluster_deny_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cluster_elb_sl_role_creation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.workers_additional_policies](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_launch_configuration.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_configuration) | resource | -| [aws_launch_template.workers_launch_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_security_group.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_security_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_security_group_rule.cluster_egress_internet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.cluster_https_worker_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.cluster_primary_ingress_workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.cluster_private_access_cidrs_source](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.cluster_private_access_sg_source](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.workers_egress_internet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.workers_ingress_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.workers_ingress_cluster_https](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.workers_ingress_cluster_kubelet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.workers_ingress_cluster_primary](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.workers_ingress_self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | -| [local_file.kubeconfig](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [aws_ami.eks_worker](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | -| [aws_ami.eks_worker_windows](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_iam_instance_profile.custom_worker_group_iam_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_instance_profile) | data source | -| [aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_instance_profile) | data source | -| [aws_iam_policy_document.cluster_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cluster_deny_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cluster_elb_sl_role_creation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.workers_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_role.custom_cluster_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | +| [aws_security_group.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | -| [http_http.wait_for_cluster](https://registry.terraform.io/providers/terraform-aws-modules/http/latest/docs/data-sources/http) | data source | +| [tls_certificate.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/data-sources/certificate) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [attach\_worker\_cni\_policy](#input\_attach\_worker\_cni\_policy) | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster. | `bool` | `true` | no | -| [aws\_auth\_additional\_labels](#input\_aws\_auth\_additional\_labels) | Additional kubernetes labels applied on aws-auth ConfigMap | `map(string)` | `{}` | no | -| [cluster\_create\_endpoint\_private\_access\_sg\_rule](#input\_cluster\_create\_endpoint\_private\_access\_sg\_rule) | Whether to create security group rules for the access to the Amazon EKS private API server endpoint. When is `true`, `cluster_endpoint_private_access_cidrs` must be setted. | `bool` | `false` | no | -| [cluster\_create\_security\_group](#input\_cluster\_create\_security\_group) | Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`. | `bool` | `true` | no | -| [cluster\_create\_timeout](#input\_cluster\_create\_timeout) | Timeout value when creating the EKS cluster. | `string` | `"30m"` | no | -| [cluster\_delete\_timeout](#input\_cluster\_delete\_timeout) | Timeout value when deleting the EKS cluster. | `string` | `"15m"` | no | -| [cluster\_egress\_cidrs](#input\_cluster\_egress\_cidrs) | List of CIDR blocks that are permitted for cluster egress traffic. | `list(string)` |
[
"0.0.0.0/0"
]
| no | -| [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no | -| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster. See examples/secrets\_encryption/main.tf for example format |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | -| [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | -| [cluster\_endpoint\_private\_access\_cidrs](#input\_cluster\_endpoint\_private\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS private API server endpoint. To use this `cluster_endpoint_private_access` and `cluster_create_endpoint_private_access_sg_rule` must be set to `true`. | `list(string)` | `null` | no | -| [cluster\_endpoint\_private\_access\_sg](#input\_cluster\_endpoint\_private\_access\_sg) | List of security group IDs which can access the Amazon EKS private API server endpoint. To use this `cluster_endpoint_private_access` and `cluster_create_endpoint_private_access_sg_rule` must be set to `true`. | `list(string)` | `null` | no | -| [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled. When it's set to `false` ensure to have a proper private access with `cluster_endpoint_private_access = true`. | `bool` | `true` | no | -| [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint. | `list(string)` |
[
"0.0.0.0/0"
]
| no | -| [cluster\_iam\_role\_name](#input\_cluster\_iam\_role\_name) | IAM role name for the cluster. If manage\_cluster\_iam\_resources is set to false, set this to reuse an existing IAM role. If manage\_cluster\_iam\_resources is set to true, set this to force the created role name. | `string` | `""` | no | -| [cluster\_log\_kms\_key\_id](#input\_cluster\_log\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `""` | no | -| [cluster\_log\_retention\_in\_days](#input\_cluster\_log\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no | -| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster. Also used as a prefix in names of related resources. | `string` | `""` | no | -| [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers | `string` | `""` | no | -| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no | -| [cluster\_tags](#input\_cluster\_tags) | A map of tags to add to just the eks resource. | `map(string)` | `{}` | no | -| [cluster\_update\_timeout](#input\_cluster\_update\_timeout) | Timeout value when updating the EKS cluster. | `string` | `"60m"` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes minor version to use for the EKS cluster (for example 1.21). | `string` | `null` | no | -| [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | -| [create\_fargate\_pod\_execution\_role](#input\_create\_fargate\_pod\_execution\_role) | Controls if the EKS Fargate pod execution IAM role should be created. | `bool` | `true` | no | -| [default\_platform](#input\_default\_platform) | Default platform name. Valid options are `linux` and `windows`. | `string` | `"linux"` | no | -| [eks\_oidc\_root\_ca\_thumbprint](#input\_eks\_oidc\_root\_ca\_thumbprint) | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | -| [enable\_irsa](#input\_enable\_irsa) | Whether to create OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no | -| [fargate\_pod\_execution\_role\_name](#input\_fargate\_pod\_execution\_role\_name) | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | -| [fargate\_profiles](#input\_fargate\_profiles) | Fargate profiles to create. See `fargate_profile` keys section in fargate submodule's README.md for more details | `any` | `{}` | no | -| [fargate\_subnets](#input\_fargate\_subnets) | A list of subnets to place fargate workers within (if different from subnets). | `list(string)` | `[]` | no | -| [iam\_path](#input\_iam\_path) | If provided, all IAM roles will be created on this path. | `string` | `"/"` | no | -| [kubeconfig\_api\_version](#input\_kubeconfig\_api\_version) | KubeConfig API version. Defaults to client.authentication.k8s.io/v1alpha1 | `string` | `"client.authentication.k8s.io/v1alpha1"` | no | -| [kubeconfig\_aws\_authenticator\_additional\_args](#input\_kubeconfig\_aws\_authenticator\_additional\_args) | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | `list(string)` | `[]` | no | -| [kubeconfig\_aws\_authenticator\_command](#input\_kubeconfig\_aws\_authenticator\_command) | Command to use to fetch AWS EKS credentials. | `string` | `"aws-iam-authenticator"` | no | -| [kubeconfig\_aws\_authenticator\_command\_args](#input\_kubeconfig\_aws\_authenticator\_command\_args) | Default arguments passed to the authenticator command. Defaults to [token -i $cluster\_name]. | `list(string)` | `[]` | no | -| [kubeconfig\_aws\_authenticator\_env\_variables](#input\_kubeconfig\_aws\_authenticator\_env\_variables) | Environment variables that should be used when executing the authenticator. e.g. { AWS\_PROFILE = "eks"}. | `map(string)` | `{}` | no | -| [kubeconfig\_file\_permission](#input\_kubeconfig\_file\_permission) | File permission of the Kubectl config file containing cluster configuration saved to `kubeconfig_output_path.` | `string` | `"0600"` | no | -| [kubeconfig\_name](#input\_kubeconfig\_name) | Override the default name used for items kubeconfig. | `string` | `""` | no | -| [kubeconfig\_output\_path](#input\_kubeconfig\_output\_path) | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | -| [manage\_aws\_auth](#input\_manage\_aws\_auth) | Whether to apply the aws-auth configmap file. | `bool` | `true` | no | -| [manage\_cluster\_iam\_resources](#input\_manage\_cluster\_iam\_resources) | Whether to let the module manage cluster IAM resources. If set to false, cluster\_iam\_role\_name must be specified. | `bool` | `true` | no | -| [manage\_worker\_iam\_resources](#input\_manage\_worker\_iam\_resources) | Whether to let the module manage worker IAM resources. If set to false, iam\_instance\_profile\_name must be specified for workers. | `bool` | `true` | no | -| [map\_accounts](#input\_map\_accounts) | Additional AWS account numbers to add to the aws-auth configmap. | `list(string)` | `[]` | no | -| [map\_roles](#input\_map\_roles) | Additional IAM roles to add to the aws-auth configmap. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
| `[]` | no | -| [map\_users](#input\_map\_users) | Additional IAM users to add to the aws-auth configmap. |
list(object({
userarn = string
username = string
groups = list(string)
}))
| `[]` | no | -| [node\_groups](#input\_node\_groups) | Map of map of node groups to create. See `node_groups` module's documentation for more details | `any` | `{}` | no | -| [node\_groups\_defaults](#input\_node\_groups\_defaults) | Map of values to be applied to all node groups. See `node_groups` module's documentation for more details | `any` | `{}` | no | -| [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider. | `list(string)` | `[]` | no | -| [permissions\_boundary](#input\_permissions\_boundary) | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | -| [subnets](#input\_subnets) | A list of subnets to place the EKS cluster and workers within. | `list(string)` | `[]` | no | -| [tags](#input\_tags) | A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only. | `map(string)` | `{}` | no | -| [vpc\_id](#input\_vpc\_id) | VPC where the cluster and workers will be deployed. | `string` | `null` | no | -| [wait\_for\_cluster\_timeout](#input\_wait\_for\_cluster\_timeout) | A timeout (in seconds) to wait for cluster to be available. | `number` | `300` | no | -| [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | -| [worker\_ami\_name\_filter](#input\_worker\_ami\_name\_filter) | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | -| [worker\_ami\_name\_filter\_windows](#input\_worker\_ami\_name\_filter\_windows) | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | -| [worker\_ami\_owner\_id](#input\_worker\_ami\_owner\_id) | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"amazon"` | no | -| [worker\_ami\_owner\_id\_windows](#input\_worker\_ami\_owner\_id\_windows) | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"amazon"` | no | -| [worker\_create\_cluster\_primary\_security\_group\_rules](#input\_worker\_create\_cluster\_primary\_security\_group\_rules) | Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group. | `bool` | `false` | no | -| [worker\_create\_initial\_lifecycle\_hooks](#input\_worker\_create\_initial\_lifecycle\_hooks) | Whether to create initial lifecycle hooks provided in worker groups. | `bool` | `false` | no | -| [worker\_create\_security\_group](#input\_worker\_create\_security\_group) | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | `bool` | `true` | no | -| [worker\_groups](#input\_worker\_groups) | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | -| [worker\_groups\_launch\_template](#input\_worker\_groups\_launch\_template) | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | -| [worker\_security\_group\_id](#input\_worker\_security\_group\_id) | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | -| [worker\_sg\_ingress\_from\_port](#input\_worker\_sg\_ingress\_from\_port) | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | `number` | `1025` | no | -| [workers\_additional\_policies](#input\_workers\_additional\_policies) | Additional policies to be added to workers | `list(string)` | `[]` | no | -| [workers\_egress\_cidrs](#input\_workers\_egress\_cidrs) | List of CIDR blocks that are permitted for workers egress traffic. | `list(string)` |
[
"0.0.0.0/0"
]
| no | -| [workers\_group\_defaults](#input\_workers\_group\_defaults) | Override default values for target groups. See workers\_group\_defaults\_defaults in local.tf for valid keys. | `any` | `{}` | no | -| [workers\_role\_name](#input\_workers\_role\_name) | User defined workers role name. | `string` | `""` | no | -| [write\_kubeconfig](#input\_write\_kubeconfig) | Whether to write a Kubectl config file containing the cluster configuration. Saved to `kubeconfig_output_path`. | `bool` | `true` | no | +| [cloudwatch\_log\_group\_kms\_key\_id](#input\_cloudwatch\_log\_group\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `null` | no | +| [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days | `number` | `90` | no | +| [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | +| [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no | +| [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | +| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | +| [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `false` | no | +| [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `true` | no | +| [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | +| [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | +| [cluster\_security\_group\_additional\_rules](#input\_cluster\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created | `map(any)` | `{}` | no | +| [cluster\_security\_group\_description](#input\_cluster\_security\_group\_description) | Description of the cluster security group created | `string` | `"EKS cluster security group"` | no | +| [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Existing security group ID to be attached to the cluster. Required if `create_cluster_security_group` = `false` | `string` | `""` | no | +| [cluster\_security\_group\_name](#input\_cluster\_security\_group\_name) | Name to use on cluster security group created | `string` | `null` | no | +| [cluster\_security\_group\_tags](#input\_cluster\_security\_group\_tags) | A map of additional tags to add to the cluster security group created | `map(string)` | `{}` | no | +| [cluster\_security\_group\_use\_name\_prefix](#input\_cluster\_security\_group\_use\_name\_prefix) | Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix | `string` | `true` | no | +| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | +| [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | +| [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.21`) | `string` | `null` | no | +| [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | +| [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | +| [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster or use the existing `cluster_security_group_id` | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether a an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | +| [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | +| [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | +| [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no | +| [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no | +| [fargate\_profiles](#input\_fargate\_profiles) | Map of Fargate Profile definitions to create | `any` | `{}` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the cluster. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | Cluster IAM role path | `string` | `null` | no | +| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | +| [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created | `map(any)` | `{}` | no | +| [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | +| [node\_security\_group\_id](#input\_node\_security\_group\_id) | ID of an existing security group to attach to the node groups created | `string` | `""` | no | +| [node\_security\_group\_name](#input\_node\_security\_group\_name) | Name to use on node security group created | `string` | `null` | no | +| [node\_security\_group\_tags](#input\_node\_security\_group\_tags) | A map of additional tags to add to the node security group created | `map(string)` | `{}` | no | +| [node\_security\_group\_use\_name\_prefix](#input\_node\_security\_group\_use\_name\_prefix) | Determines whether node security group name (`node_security_group_name`) is used as a prefix | `string` | `true` | no | +| [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider | `list(string)` | `[]` | no | +| [self\_managed\_node\_group\_defaults](#input\_self\_managed\_node\_group\_defaults) | Map of self-managed node group default configurations | `any` | `{}` | no | +| [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create | `any` | `{}` | no | +| [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the EKS cluster (ENIs) will be provisioned along with the nodes/node groups. Node groups can be deployed within a different set of subnet IDs from within the node group configuration | `list(string)` | `[]` | no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | +| [vpc\_id](#input\_vpc\_id) | ID of the VPC where the cluster and its nodes will be provisioned | `string` | `null` | no | ## Outputs | Name | Description | |------|-------------| +| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | | [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | | [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | -| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster. | -| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | -| [cluster\_endpoint](#output\_cluster\_endpoint) | The endpoint for your EKS Kubernetes API. | -| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster. | -| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster. | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready. | -| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster OIDC Issuer | -| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | The cluster primary security group ID created by the EKS cluster on 1.14 or later. Referred to as 'Cluster security group' in the EKS console. | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console. | -| [cluster\_version](#output\_cluster\_version) | The Kubernetes server version for the EKS cluster. | -| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | -| [fargate\_iam\_role\_arn](#output\_fargate\_iam\_role\_arn) | IAM role ARN for EKS Fargate pods | -| [fargate\_iam\_role\_name](#output\_fargate\_iam\_role\_name) | IAM role name for EKS Fargate pods | -| [fargate\_profile\_arns](#output\_fargate\_profile\_arns) | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | -| [fargate\_profile\_ids](#output\_fargate\_profile\_ids) | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | -| [kubeconfig](#output\_kubeconfig) | kubectl config file contents for this EKS cluster. Will block on cluster creation until the cluster is really ready. | -| [kubeconfig\_filename](#output\_kubeconfig\_filename) | The filename of the generated kubectl config. Will block on cluster creation until the cluster is really ready. | -| [node\_groups](#output\_node\_groups) | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys | -| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true`. | -| [security\_group\_rule\_cluster\_https\_worker\_ingress](#output\_security\_group\_rule\_cluster\_https\_worker\_ingress) | Security group rule responsible for allowing pods to communicate with the EKS cluster API. | -| [worker\_iam\_instance\_profile\_arns](#output\_worker\_iam\_instance\_profile\_arns) | default IAM instance profile ARN for EKS worker groups | -| [worker\_iam\_instance\_profile\_names](#output\_worker\_iam\_instance\_profile\_names) | default IAM instance profile name for EKS worker groups | -| [worker\_iam\_role\_arn](#output\_worker\_iam\_role\_arn) | default IAM role ARN for EKS worker groups | -| [worker\_iam\_role\_name](#output\_worker\_iam\_role\_name) | default IAM role name for EKS worker groups | -| [worker\_security\_group\_id](#output\_worker\_security\_group\_id) | Security group ID attached to the EKS workers. | -| [workers\_asg\_arns](#output\_workers\_asg\_arns) | IDs of the autoscaling groups containing workers. | -| [workers\_asg\_names](#output\_workers\_asg\_names) | Names of the autoscaling groups containing workers. | -| [workers\_default\_ami\_id](#output\_workers\_default\_ami\_id) | ID of the default worker group AMI | -| [workers\_default\_ami\_id\_windows](#output\_workers\_default\_ami\_id\_windows) | ID of the default Windows worker group AMI | -| [workers\_launch\_template\_arns](#output\_workers\_launch\_template\_arns) | ARNs of the worker launch templates. | -| [workers\_launch\_template\_ids](#output\_workers\_launch\_template\_ids) | IDs of the worker launch templates. | -| [workers\_launch\_template\_latest\_versions](#output\_workers\_launch\_template\_latest\_versions) | Latest versions of the worker launch templates. | -| [workers\_user\_data](#output\_workers\_user\_data) | User data of worker groups | +| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | +| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | +| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | +| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | +| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | +| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | +| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | + +## License + +Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-rds-aurora/tree/master/LICENSE) for full details. diff --git a/UPGRADE-18.0.md b/UPGRADE-18.0.md new file mode 100644 index 0000000000..0c6d56dde6 --- /dev/null +++ b/UPGRADE-18.0.md @@ -0,0 +1,550 @@ +# Upgrade from v17.x to v18.x + +Please consult the `examples` directory for reference example configurations. If you find a bug, please open an issue with supporting configuration to reproduce. + +## List of backwards incompatible changes + +- Launch configuration support has been removed and only launch template is supported going forward. AWS is no longer adding new features back into launch configuration and their docs state [`We strongly recommend that you do not use launch configurations. They do not provide full functionality for Amazon EC2 Auto Scaling or Amazon EC2. We provide information about launch configurations for customers who have not yet migrated from launch configurations to launch templates.`](https://docs.aws.amazon.com/autoscaling/ec2/userguide/LaunchConfiguration.html) +- Support for managing aws-auth configmap has been removed. This change also removes the dependency on the Kubernetes Terraform provider, the local dependency on aws-iam-authenticator for users, as well as the reliance on the forked http provider to wait and poll on cluster creation. To aid users in this change, an output variable `aws_auth_configmap_yaml` has been provided which renders the aws-auth configmap necessary to support at least the IAM roles used by the module (additional mapRoles/mapUsers definitions to be provided by users) +- Support for managing kubeconfig and its associated `local_file` resources have been removed; users are able to use the awscli provided `aws eks update-kubeconfig --name ` to update their local kubeconfig as necessary +- The terminology used in the module has been modified to reflect that used by the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/eks-compute.html). + - [AWS EKS Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html), `eks_managed_node_groups`, was previously referred to as simply node group, `node_groups` + - [Self Managed Node Group Group](https://docs.aws.amazon.com/eks/latest/userguide/worker.html), `self_managed_node_groups`, was previously referred to as worker group, `worker_groups` + - [AWS Fargate Profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html), `fargate_profiles`, remains unchanged in terms of naming and terminology +- The three different node group types supported by AWS and the module have been refactored into standalone sub-modules that are both used by the root `eks` module as well as available for individual, standalone consumption if desired. + - The previous `node_groups` sub-module is now named `eks-managed-node-group` and provisions a single AWS EKS Managed Node Group per sub-module definition (previous version utilized `for_each` to create 0 or more node groups) + - Additional changes for the `eks-managed-node-group` sub-module over the previous `node_groups` module include: + - Variable name changes defined in section `Variable and output changes` below + - Support for nearly full control of the IAM role created, or provide the ARN of an existing IAM role, has been added + - Support for nearly full control of the security group created, or provide the ID of an existing security group, has been added + - User data has been revamped and all user data logic moved to the `_user_data` internal sub-module; the local `userdata.sh.tpl` has been removed entirely + - The previous `fargate` sub-module is now named `fargate-profile` and provisions a single AWS EKS Fargate Profile per sub-module definition (previous version utilized `for_each` to create 0 or more profiles) + - Additional changes for the `fargate-profile` sub-module over the previous `fargate` module include: + - Variable name changes defined in section `Variable and output changes` below + - Support for nearly full control of the IAM role created, or provide the ARN of an existing IAM role, has been added + - Similar to the `eks_managed_node_group_defaults` and `self_managed_node_group_defaults`, a `fargate_profile_defaults` has been provided to allow users to control the default configurations for the Fargate profiles created + - A sub-module for `self-managed-node-group` has been created and provisions a single self managed node group (autoscaling group) per sub-module definition + - Additional changes for the `self-managed-node-group` sub-module over the previous `node_groups` variable include: + - The underlying autoscaling group and launch template have been updated to more closely match that of the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module and the features it offers + - The previous iteration used a count over a list of node group definitions which was prone to disruptive updates; this is now replaced with a map/for_each to align with that of the EKS managed node group and Fargate profile behaviors/style +- The user data configuration supported across the module has been completely revamped. A new `_user_data` internal sub-module has been created to consolidate all user data configuration in one location which provides better support for testability (via the [`examples/user_data`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) example). The new sub-module supports nearly all possible combinations including the ability to allow users to provide their own user data template which will be rendered by the module. See the `examples/user_data` example project for the full plethora of example configuration possibilities and more details on the logic of the design can be found in the [`modules/_user_data`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/_user_data_) directory. + +## Additional changes + +### Added + +- Support for AWS EKS Addons has been added +- Support for AWS EKS Cluster Identity Provider Configuration has been added +- AWS Terraform provider minimum required version has been updated to 3.64 to support the changes made and additional resources supported +- An example `user_data` project has been added to aid in demonstrating, testing, and validating the various methods of configuring user data with the `_user_data` sub-module as well as the root `eks` module +- Template for rendering the aws-auth configmap output - `aws_auth_cm.tpl` +- Template for Bottlerocket OS user data bootstrapping - `bottlerocket_user_data.tpl` + +### Modified + +- The previous `fargate` example has been renamed to `fargate_profile` +- The previous `irsa` and `instance_refresh` examples have been merged into one example `irsa_autoscale_refresh` +- The previous `managed_node_groups` example has been renamed to `self_managed_node_group` +- The previously hardcoded EKS OIDC root CA thumbprint value and variable has been replaced with a `tls_certificate` data source that refers to the cluster OIDC issuer url. Thumbprint values should remain unchanged however +- Individual cluster security group resources have been replaced with a single security group resource that takes a map of rules as input. The default ingress/egress rules have had their scope reduced in order to provide the bare minimum of access to permit successful cluster creation and allow users to opt in to any additional network access as needed for a better security posture. This means the `0.0.0.0/0` egress rule has been removed, instead TCP/443 and TCP/10250 egress rules to the node group security group are used instead +- The Linux/bash user data template has been updated to include the bare minimum necessary for bootstrapping AWS EKS Optimized AMI derivative nodes with provisions for providing additional user data and configurations; was named `userdata.sh.tpl` and is now named `linux_user_data.tpl` +- The Windows user data template has been renamed from `userdata_windows.tpl` to `windows_user_data.tpl` + +### Removed + +- Miscellaneous documents on how to configure Kubernetes cluster internals have been removed. Documentation related to how to configure the AWS EKS Cluster and its supported infrastructure resources provided by the module are supported, while cluster internal configuration is out of scope for this project +- The previous `bottlerocket` example has been removed in favor of demonstrating the use and configuration of Bottlerocket nodes via the respective `eks_managed_node_group` and `self_managed_node_group` examples +- The previous `launch_template` and `launch_templates_with_managed_node_groups` examples have been removed; only launch templates are now supported (default) and launch configuration support has been removed +- The previous `secrets_encryption` example has been removed; the functionality has been demonstrated in several of the new examples rendering this standalone example redundant +- The additional, custom IAM role policy for the cluster role has been removed. The permissions are either now provided in the attached managed AWS permission policies used or are no longer required +- The `kubeconfig.tpl` template; kubeconfig management is no longer supported under this module +- The HTTP Terraform provider (forked copy) dependency has been removed + +### Variable and output changes + +1. Removed variables: + + - `cluster_create_timeout`, `cluster_update_timeout`, and `cluster_delete_timeout` have been replaced with `cluster_timeouts` + - `kubeconfig_name` + - `kubeconfig_output_path` + - `kubeconfig_file_permission` + - `kubeconfig_api_version` + - `kubeconfig_aws_authenticator_command` + - `kubeconfig_aws_authenticator_command_args` + - `kubeconfig_aws_authenticator_additional_args` + - `kubeconfig_aws_authenticator_env_variables` + - `write_kubeconfig` + - `default_platform` + - `manage_aws_auth` + - `aws_auth_additional_labels` + - `map_accounts` + - `map_roles` + - `map_users` + - `fargate_subnets` + - `worker_groups_launch_template` + - `worker_security_group_id` + - `worker_ami_name_filter` + - `worker_ami_name_filter_windows` + - `worker_ami_owner_id` + - `worker_ami_owner_id_windows` + - `worker_additional_security_group_ids` + - `worker_sg_ingress_from_port` + - `workers_additional_policies` + - `worker_create_security_group` + - `worker_create_initial_lifecycle_hooks` + - `worker_create_cluster_primary_security_group_rules` + - `cluster_create_endpoint_private_access_sg_rule` + - `cluster_endpoint_private_access_cidrs` + - `cluster_endpoint_private_access_sg` + - `manage_worker_iam_resources` + - `workers_role_name` + - `attach_worker_cni_policy` + - `eks_oidc_root_ca_thumbprint` + - `create_fargate_pod_execution_role` + - `fargate_pod_execution_role_name` + - `cluster_egress_cidrs` + - `workers_egress_cidrs` + - `wait_for_cluster_timeout` + - EKS Managed Node Group sub-module (was `node_groups`) + - `default_iam_role_arn` + - `workers_group_defaults` + - `worker_security_group_id` + - `node_groups_defaults` + - `node_groups` + - `ebs_optimized_not_supported` + - Fargate profile sub-module (was `fargate`) + - `create_eks` and `create_fargate_pod_execution_role` have been replaced with simply `create` + +2. Renamed variables: + + - `create_eks` -> `create` + - `subnets` -> `subnet_ids` + - `cluster_create_security_group` -> `create_cluster_security_group` + - `cluster_log_retention_in_days` -> `cloudwatch_log_group_retention_in_days` + - `cluster_log_kms_key_id` -> `cloudwatch_log_group_kms_key_id` + - `manage_cluster_iam_resources` -> `create_iam_role` + - `cluster_iam_role_name` -> `iam_role_name` + - `permissions_boundary` -> `iam_role_permissions_boundary` + - `iam_path` -> `iam_role_path` + - `pre_userdata` -> `pre_bootstrap_user_data` + - `additional_userdata` -> `post_bootstrap_user_data` + - `worker_groups` -> `self_managed_node_groups` + - `workers_group_defaults` -> `self_managed_node_group_defaults` + - `node_groups` -> `eks_managed_node_groups` + - `node_groups_defaults` -> `eks_managed_node_group_defaults` + - EKS Managed Node Group sub-module (was `node_groups`) + - `create_eks` -> `create` + - `worker_additional_security_group_ids` -> `vpc_security_group_ids` + - Fargate profile sub-module + - `fargate_pod_execution_role_name` -> `name` + - `create_fargate_pod_execution_role` -> `create_iam_role` + - `subnets` -> `subnet_ids` + - `iam_path` -> `iam_role_path` + - `permissions_boundary` -> `iam_role_permissions_boundary` + +3. Added variables: + + - `cluster_additional_security_group_ids` added to allow users to add additional security groups to the cluster as needed + - `cluster_security_group_name` + - `cluster_security_group_use_name_prefix` added to allow users to use either the name as specified or default to using the name specified as a prefix + - `cluster_security_group_description` + - `cluster_security_group_additional_rules` + - `cluster_security_group_tags` + - `create_cloudwatch_log_group` added in place of the logic that checked if any cluster log types were enabled to allow users to opt in as they see fit + - `create_node_security_group` added to create single security group that connects node groups and cluster in central location + - `node_security_group_id` + - `node_security_group_name` + - `node_security_group_use_name_prefix` + - `node_security_group_description` + - `node_security_group_additional_rules` + - `node_security_group_tags` + - `iam_role_arn` + - `iam_role_use_name_prefix` + - `iam_role_description` + - `iam_role_additional_policies` + - `iam_role_tags` + - `cluster_addons` + - `cluster_identity_providers` + - `fargate_profile_defaults` + - EKS Managed Node Group sub-module (was `node_groups`) + - `platform` + - `enable_bootstrap_user_data` + - `pre_bootstrap_user_data` + - `post_bootstrap_user_data` + - `bootstrap_extra_args` + - `user_data_template_path` + - `create_launch_template` + - `launch_template_name` + - `launch_template_use_name_prefix` + - `description` + - `ebs_optimized` + - `ami_id` + - `key_name` + - `launch_template_default_version` + - `update_launch_template_default_version` + - `disable_api_termination` + - `kernel_id` + - `ram_disk_id` + - `block_device_mappings` + - `capacity_reservation_specification` + - `cpu_options` + - `credit_specification` + - `elastic_gpu_specifications` + - `elastic_inference_accelerator` + - `enclave_options` + - `instance_market_options` + - `license_specifications` + - `metadata_options` + - `enable_monitoring` + - `network_interfaces` + - `placement` + - `min_size` + - `max_size` + - `desired_size` + - `use_name_prefix` + - `ami_type` + - `ami_release_version` + - `capacity_type` + - `disk_size` + - `force_update_version` + - `instance_types` + - `labels` + - `cluster_version` + - `launch_template_version` + - `remote_access` + - `taints` + - `update_config` + - `timeouts` + - `create_security_group` + - `security_group_name` + - `security_group_use_name_prefix` + - `security_group_description` + - `vpc_id` + - `security_group_rules` + - `cluster_security_group_id` + - `security_group_tags` + - `create_iam_role` + - `iam_role_arn` + - `iam_role_name` + - `iam_role_use_name_prefix` + - `iam_role_path` + - `iam_role_description` + - `iam_role_permissions_boundary` + - `iam_role_additional_policies` + - `iam_role_tags` + - Fargate profile sub-module (was `fargate`) + - `iam_role_arn` (for if `create_iam_role` is `false` to bring your own externally created role) + - `iam_role_name` + - `iam_role_use_name_prefix` + - `iam_role_description` + - `iam_role_additional_policies` + - `iam_role_tags` + - `selectors` + - `timeouts` + +4. Removed outputs: + + - `cluster_version` + - `kubeconfig` + - `kubeconfig_filename` + - `workers_asg_arns` + - `workers_asg_names` + - `workers_user_data` + - `workers_default_ami_id` + - `workers_default_ami_id_windows` + - `workers_launch_template_ids` + - `workers_launch_template_arns` + - `workers_launch_template_latest_versions` + - `worker_security_group_id` + - `worker_iam_instance_profile_arns` + - `worker_iam_instance_profile_names` + - `worker_iam_role_name` + - `worker_iam_role_arn` + - `fargate_profile_ids` + - `fargate_profile_arns` + - `fargate_iam_role_name` + - `fargate_iam_role_arn` + - `node_groups` + - `security_group_rule_cluster_https_worker_ingress` + - EKS Managed Node Group sub-module (was `node_groups`) + - `node_groups` + - `aws_auth_roles` + - Fargate profile sub-module (was `fargate`) + - `aws_auth_roles` + +5. Renamed outputs: + + - `config_map_aws_auth` -> `aws_auth_configmap_yaml` + - Fargate profile sub-module (was `fargate`) + - `fargate_profile_ids` -> `fargate_profile_id` + - `fargate_profile_arns` -> `fargate_profile_arn` + +6. Added outputs: + + - `cluster_platform_version` + - `cluster_status` + - `cluster_security_group_arn` + - `cluster_security_group_id` + - `node_security_group_arn` + - `node_security_group_id` + - `cluster_iam_role_unique_id` + - `cluster_addons` + - `cluster_identity_providers` + - `fargate_profiles` + - `eks_managed_node_groups` + - `self_managed_node_groups` + - EKS Managed Node Group sub-module (was `node_groups`) + - `launch_template_id` + - `launch_template_arn` + - `launch_template_latest_version` + - `node_group_arn` + - `node_group_id` + - `node_group_resources` + - `node_group_status` + - `security_group_arn` + - `security_group_id` + - `iam_role_name` + - `iam_role_arn` + - `iam_role_unique_id` + - Fargate profile sub-module (was `fargate`) + - `iam_role_unique_id` + - `fargate_profile_status` + +## Upgrade Migrations + +### Before 17.x Example + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 17.0" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnets = module.vpc.private_subnets + + # Managed Node Groups + node_groups_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + } + + node_groups = { + node_group = { + min_capacity = 1 + max_capacity = 10 + desired_capacity = 1 + + instance_types = ["t3.large"] + capacity_type = "SPOT" + + update_config = { + max_unavailable_percentage = 50 + } + + k8s_labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + taints = [ + { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + ] + + additional_tags = { + ExtraTag = "example" + } + } + } + + # Worker groups + worker_additional_security_group_ids = [aws_security_group.additional.id] + + worker_groups_launch_template = [ + { + name = "worker-group" + override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] + spot_instance_pools = 4 + asg_max_size = 5 + asg_desired_capacity = 2 + kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" + public_ip = true + }, + ] + + # Fargate + fargate_profiles = { + default = { + name = "default" + selectors = [ + { + namespace = "kube-system" + labels = { + k8s-app = "kube-dns" + } + }, + { + namespace = "default" + } + ] + + tags = { + Owner = "test" + } + + timeouts = { + create = "20m" + delete = "20m" + } + } + } + + tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} +``` + +### After 18.x Example + +```hcl +module "cluster_after" { + source = "terraform-aws-modules/eks/aws" + version = "~> 18.0" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + } + + eks_managed_node_groups = { + node_group = { + min_size = 1 + max_size = 10 + desired_size = 1 + + instance_types = ["t3.large"] + capacity_type = "SPOT" + + update_config = { + max_unavailable_percentage = 50 + } + + labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + taints = [ + { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + ] + + tags = { + ExtraTag = "example" + } + } + } + + self_managed_node_group_defaults = { + vpc_security_group_ids = [aws_security_group.additional.id] + } + + self_managed_node_groups = { + worker_group = { + name = "worker-group" + + min_size = 1 + max_size = 5 + desired_size = 2 + instance_type = "m4.large" + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + delete_on_termination = true + encrypted = false + volume_size = 100 + volume_type = "gp2" + } + + } + } + + use_mixed_instances_policy = true + mixed_instances_policy = { + instances_distribution = { + spot_instance_pools = 4 + } + + override = [ + { instance_type = "m5.large" }, + { instance_type = "m5a.large" }, + { instance_type = "m5d.large" }, + { instance_type = "m5ad.large" }, + ] + } + } + } + + # Fargate + fargate_profiles = { + default = { + name = "default" + + selectors = [ + { + namespace = "kube-system" + labels = { + k8s-app = "kube-dns" + } + }, + { + namespace = "default" + } + ] + + tags = { + Owner = "test" + } + + timeouts = { + create = "20m" + delete = "20m" + } + } + } + + tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} +``` diff --git a/aws_auth.tf b/aws_auth.tf deleted file mode 100644 index 6de697607f..0000000000 --- a/aws_auth.tf +++ /dev/null @@ -1,92 +0,0 @@ -locals { - auth_launch_template_worker_roles = [ - for index in range(0, var.create_eks ? local.worker_group_launch_template_count : 0) : { - worker_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${element( - coalescelist( - aws_iam_instance_profile.workers_launch_template.*.role, - data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, - [""] - ), - index - )}" - platform = lookup( - var.worker_groups_launch_template[index], - "platform", - local.workers_group_defaults["platform"] - ) - } - ] - - auth_worker_roles = [ - for index in range(0, var.create_eks ? local.worker_group_launch_configuration_count : 0) : { - worker_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${element( - coalescelist( - aws_iam_instance_profile.workers.*.role, - data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, - [""] - ), - index, - )}" - platform = lookup( - var.worker_groups[index], - "platform", - local.workers_group_defaults["platform"] - ) - } - ] - - # Convert to format needed by aws-auth ConfigMap - configmap_roles = [ - for role in concat( - local.auth_launch_template_worker_roles, - local.auth_worker_roles, - module.node_groups.aws_auth_roles, - module.fargate.aws_auth_roles, - ) : - { - # Work around https://github.com/kubernetes-sigs/aws-iam-authenticator/issues/153 - # Strip the leading slash off so that Terraform doesn't think it's a regex - rolearn = replace(role["worker_role_arn"], replace(var.iam_path, "/^//", ""), "") - username = role["platform"] == "fargate" ? "system:node:{{SessionName}}" : "system:node:{{EC2PrivateDNSName}}" - groups = tolist(concat( - [ - "system:bootstrappers", - "system:nodes", - ], - role["platform"] == "windows" ? ["eks:kube-proxy-windows"] : [], - role["platform"] == "fargate" ? ["system:node-proxier"] : [], - )) - } - ] -} - -resource "kubernetes_config_map" "aws_auth" { - count = var.create_eks && var.manage_aws_auth ? 1 : 0 - - metadata { - name = "aws-auth" - namespace = "kube-system" - labels = merge( - { - "app.kubernetes.io/managed-by" = "Terraform" - # / are replaced by . because label validator fails in this lib - # https://github.com/kubernetes/apimachinery/blob/1bdd76d09076d4dc0362456e59c8f551f5f24a72/pkg/util/validation/validation.go#L166 - "terraform.io/module" = "terraform-aws-modules.eks.aws" - }, - var.aws_auth_additional_labels - ) - } - - data = { - mapRoles = yamlencode( - distinct(concat( - local.configmap_roles, - var.map_roles, - )) - ) - mapUsers = yamlencode(var.map_users) - mapAccounts = yamlencode(var.map_accounts) - } - - depends_on = [data.http.wait_for_cluster[0]] -} diff --git a/data.tf b/data.tf deleted file mode 100644 index fe3b6e33c7..0000000000 --- a/data.tf +++ /dev/null @@ -1,104 +0,0 @@ -data "aws_partition" "current" {} - -data "aws_caller_identity" "current" {} - -data "aws_iam_policy_document" "workers_assume_role_policy" { - statement { - sid = "EKSWorkerAssumeRole" - - actions = [ - "sts:AssumeRole", - ] - - principals { - type = "Service" - identifiers = [local.ec2_principal] - } - } -} - -data "aws_ami" "eks_worker" { - count = contains(local.worker_groups_platforms, "linux") ? 1 : 0 - - filter { - name = "name" - values = [local.worker_ami_name_filter] - } - - most_recent = true - - owners = [var.worker_ami_owner_id] -} - -data "aws_ami" "eks_worker_windows" { - count = contains(local.worker_groups_platforms, "windows") ? 1 : 0 - - filter { - name = "name" - values = [local.worker_ami_name_filter_windows] - } - - filter { - name = "platform" - values = ["windows"] - } - - most_recent = true - - owners = [var.worker_ami_owner_id_windows] -} - -data "aws_iam_policy_document" "cluster_assume_role_policy" { - statement { - sid = "EKSClusterAssumeRole" - - actions = [ - "sts:AssumeRole", - ] - - principals { - type = "Service" - identifiers = ["eks.amazonaws.com"] - } - } -} - -data "aws_iam_role" "custom_cluster_iam_role" { - count = var.manage_cluster_iam_resources ? 0 : 1 - - name = var.cluster_iam_role_name -} - -data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" { - count = var.manage_worker_iam_resources ? 0 : local.worker_group_launch_configuration_count - - name = lookup( - var.worker_groups[count.index], - "iam_instance_profile_name", - local.workers_group_defaults["iam_instance_profile_name"], - ) -} - -data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instance_profile" { - count = var.manage_worker_iam_resources ? 0 : local.worker_group_launch_template_count - - name = lookup( - var.worker_groups_launch_template[count.index], - "iam_instance_profile_name", - local.workers_group_defaults["iam_instance_profile_name"], - ) -} - -data "http" "wait_for_cluster" { - count = var.create_eks && var.manage_aws_auth ? 1 : 0 - - url = format("%s/healthz", aws_eks_cluster.this[0].endpoint) - ca_certificate = base64decode(local.cluster_auth_base64) - timeout = var.wait_for_cluster_timeout - - depends_on = [ - aws_eks_cluster.this, - aws_security_group_rule.cluster_private_access_sg_source, - aws_security_group_rule.cluster_private_access_cidrs_source, - ] -} diff --git a/docs/autoscaling.md b/docs/autoscaling.md deleted file mode 100644 index 3c1aa5ee93..0000000000 --- a/docs/autoscaling.md +++ /dev/null @@ -1,98 +0,0 @@ -# Autoscaling - -To enable worker node autoscaling you will need to do a few things: - -- Add the [required tags](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#auto-discovery-setup) to the worker group -- Install the cluster-autoscaler -- Give the cluster-autoscaler access via an IAM policy - -It's probably easiest to follow the example in [examples/irsa](../examples/irsa), this will install the cluster-autoscaler using [Helm](https://helm.sh/) and use IRSA to attach a policy. - -If you don't want to use IRSA then you will need to attach the IAM policy to the worker node IAM role or add AWS credentials to the cluster-autoscaler environment variables. Here is some example terraform code for the policy: - -```hcl -resource "aws_iam_role_policy_attachment" "workers_autoscaling" { - policy_arn = aws_iam_policy.worker_autoscaling.arn - role = module.my_cluster.worker_iam_role_name -} - -resource "aws_iam_policy" "worker_autoscaling" { - name_prefix = "eks-worker-autoscaling-${module.my_cluster.cluster_id}" - description = "EKS worker node autoscaling policy for cluster ${module.my_cluster.cluster_id}" - policy = data.aws_iam_policy_document.worker_autoscaling.json - path = var.iam_path - tags = var.tags -} - -data "aws_iam_policy_document" "worker_autoscaling" { - statement { - sid = "eksWorkerAutoscalingAll" - effect = "Allow" - - actions = [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "ec2:DescribeLaunchTemplateVersions", - ] - - resources = ["*"] - } - - statement { - sid = "eksWorkerAutoscalingOwn" - effect = "Allow" - - actions = [ - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - ] - - resources = ["*"] - - condition { - test = "StringEquals" - variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${module.my_cluster.cluster_id}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled" - values = ["true"] - } - } -} -``` - -And example values for the [helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler): - -```yaml -rbac: - create: true - -cloudProvider: aws -awsRegion: YOUR_AWS_REGION - -autoDiscovery: - clusterName: YOUR_CLUSTER_NAME - enabled: true - -image: - repository: us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler - tag: v1.16.5 -``` - -To install the chart, simply run helm with the `--values` option: - -``` -helm install stable/cluster-autoscaler --values=path/to/your/values-file.yaml -``` - -## Notes - -There is a variable `asg_desired_capacity` given in the `local.tf` file, currently it can be used to change the desired worker(s) capacity in the autoscaling group but currently it is being ignored in terraform to reduce the [complexities](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/510#issuecomment-531700442) and the feature of scaling up and down the cluster nodes is being handled by the cluster autoscaler. - -The cluster autoscaler major and minor versions must match your cluster. For example if you are running a 1.16 EKS cluster set `image.tag=v1.16.5`. Search through their [releases page](https://github.com/kubernetes/autoscaler/releases) for valid version numbers. diff --git a/docs/enable-docker-bridge-network.md b/docs/enable-docker-bridge-network.md deleted file mode 100644 index f6eb8ee11e..0000000000 --- a/docs/enable-docker-bridge-network.md +++ /dev/null @@ -1,23 +0,0 @@ -# Enable Docker Bridge Network - -The latest versions of the AWS EKS-optimized AMI disable the docker bridge network by default. To enable it, add the `bootstrap_extra_args` parameter to your worker group template. - -```hcl -locals { - worker_groups = [ - { - # Other parameters omitted for brevity - bootstrap_extra_args = "--enable-docker-bridge true" - } - ] -} -``` - -Examples of when this would be necessary are: - -- You are running Continuous Integration in K8s, and building docker images by either mounting the docker sock as a volume or using docker in docker. Without the bridge enabled, internal routing from the inner container can't reach the outside world. - -## See More - -- [Docker in Docker no longer works without docker0 bridge](https://github.com/awslabs/amazon-eks-ami/issues/183) -- [Add enable-docker-bridge bootstrap argument](https://github.com/awslabs/amazon-eks-ami/pull/187) diff --git a/docs/faq.md b/docs/faq.md deleted file mode 100644 index d1a5d0f4bc..0000000000 --- a/docs/faq.md +++ /dev/null @@ -1,235 +0,0 @@ -# Frequently Asked Questions - -## How do I customize X on the worker group's settings? - -All the options that can be customized for worker groups are listed in [local.tf](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/local.tf) under `workers_group_defaults_defaults`. - -Please open Issues or PRs if you think something is missing. - -## Why are nodes not being registered? - -### Networking - -Often caused by a networking or endpoint configuration issue. - -At least one of the cluster public or private endpoints must be enabled to access the cluster to work. If you require a public endpoint, setting up both (public and private) and restricting the public endpoint via setting `cluster_endpoint_public_access_cidrs` is recommended. More about communication with an endpoint is available [here](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html). - -Nodes need to be able to contact the EKS cluster endpoint. By default, the module only creates a public endpoint. To access endpoint, the nodes need outgoing internet access: - -- Nodes in private subnets: via a NAT gateway or instance. It will need adding along with appropriate routing rules. -- Nodes in public subnets: assign public IPs to nodes. Set `public_ip = true` in the `worker_groups` list on this module. - -> Important: -> If you apply only the public endpoint and setup `cluster_endpoint_public_access_cidrs` to restrict access, remember that EKS nodes also use the public endpoint, so you must allow access to the endpoint. If not, then your nodes will not be working correctly. - -Cluster private endpoint can also be enabled by setting `cluster_endpoint_private_access = true` on this module. Node calls to the endpoint stay within the VPC. - -When the private endpoint is enabled ensure that VPC DNS resolution and hostnames are also enabled: - -- If managing the VPC with Terraform: set `enable_dns_hostnames = true` and `enable_dns_support = true` on the `aws_vpc` resource. The [`terraform-aws-module/vpc/aws`](https://github.com/terraform-aws-modules/terraform-aws-vpc/) community module also has these variables. -- Otherwise refer to the [AWS VPC docs](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html#vpc-dns-updating) and [AWS EKS Cluster Endpoint Access docs](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html) for more information. - -Nodes need to be able to connect to other AWS services plus pull down container images from repos. If for some reason you cannot enable public internet access for nodes you can add VPC endpoints to the relevant services: EC2 API, ECR API, ECR DKR and S3. - -### `aws-auth` ConfigMap not present - -The module configures the `aws-auth` ConfigMap. This is used by the cluster to grant IAM users and roles RBAC permissions in the cluster, like the IAM role assigned to the worker nodes. - -Confirm that the ConfigMap matches the contents of the `config_map_aws_auth` module output. You can retrieve the live config by running the following in your terraform folder: -`kubectl --kubeconfig=kubeconfig_* -n kube-system get cm aws-auth -o yaml` - -If the ConfigMap is missing or the contents are incorrect then ensure that you have properly configured the kubernetes provider block by referring to [README.md](https://github.com/terraform-aws-modules/terraform-aws-eks/#usage-example) and run `terraform apply` again. - -Users with `manage_aws_auth = false` will need to apply the ConfigMap themselves. - -## How can I work with the cluster if I disable the public endpoint? - -You have to interact with the cluster from within the VPC that it's associated with, from an instance that's allowed access via the cluster's security group. - -Creating a new cluster with the public endpoint disabled is harder to achieve. You will either want to pass in a pre-configured cluster security group or apply the `aws-auth` configmap in a separate action. - -## ConfigMap "aws-auth" already exists - -This can happen if the kubernetes provider has not been configured for use with the cluster. The kubernetes provider will be accessing your default kubernetes cluster which already has the map defined. Read [README.md](https://github.com/terraform-aws-modules/terraform-aws-eks/#usage-example) for more details on how to configure the kubernetes provider correctly. - -Users upgrading from modules before 8.0.0 will need to import their existing aws-auth ConfigMap in to the terraform state. See 8.0.0's [CHANGELOG](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/v8.0.0/CHANGELOG.md#v800---2019-12-11) for more details. - -## `Error: Get http://localhost/api/v1/namespaces/kube-system/configmaps/aws-auth: dial tcp 127.0.0.1:80: connect: connection refused` - -Usually this means that the kubernetes provider has not been configured, there is no default `~/.kube/config` and so the kubernetes provider is attempting to talk to localhost. - -You need to configure the kubernetes provider correctly. See [README.md](https://github.com/terraform-aws-modules/terraform-aws-eks/#usage-example) for more details. - -## How can I stop Terraform from removing the EKS tags from my VPC and subnets? - -You need to add the tags to the VPC and subnets yourself. See the [basic example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). - -An alternative is to use the aws provider's [`ignore_tags` variable](https://www.terraform.io/docs/providers/aws/#ignore\_tags-configuration-block). However this can also cause terraform to display a perpetual difference. - -## How do I safely remove old worker groups? - -You've added new worker groups. Deleting worker groups from earlier in the list causes Terraform to want to recreate all worker groups. This is a limitation with how Terraform works and the module using `count` to create the ASGs and other resources. - -The safest and easiest option is to set `asg_min_size` and `asg_max_size` to 0 on the worker groups to "remove". - -## Why does changing the node or worker group's desired count not do anything? - -The module is configured to ignore this value. Unfortunately, Terraform does not support variables within the `lifecycle` block. - -The setting is ignored to allow the cluster autoscaler to work correctly so that `terraform apply` does not accidentally remove running workers. - -You can change the desired count via the CLI or console if you're not using the cluster autoscaler. - -If you are not using autoscaling and want to control the number of nodes via terraform, set the `min_capacity` and `max_capacity` for node groups or `asg_min_size` and `asg_max_size` for worker groups. Before changing those values, you must satisfy AWS `desired` capacity constraints (which must be between new min/max values). - -When you scale down AWS will remove a random instance, so you will have to weigh the risks here. - -## Why are nodes not recreated when the `launch_configuration`/`launch_template` is recreated? - -By default the ASG is not configured to be recreated when the launch configuration or template changes. Terraform spins up new instances and then deletes all the old instances in one go as the AWS provider team have refused to implement rolling updates of autoscaling groups. This is not good for kubernetes stability. - -You need to use a process to drain and cycle the workers. - -You are not using the cluster autoscaler: - -- Add a new instance -- Drain an old node `kubectl drain --force --ignore-daemonsets --delete-local-data ip-xxxxxxx.eu-west-1.compute.internal` -- Wait for pods to be Running -- Terminate the old node instance. ASG will start a new instance -- Repeat the drain and delete process until all old nodes are replaced - -You are using the cluster autoscaler: - -- Drain an old node `kubectl drain --force --ignore-daemonsets --delete-local-data ip-xxxxxxx.eu-west-1.compute.internal` -- Wait for pods to be Running -- Cluster autoscaler will create new nodes when required -- Repeat until all old nodes are drained -- Cluster autoscaler will terminate the old nodes after 10-60 minutes automatically - -You can also use a 3rd party tool like Gruntwork's kubergrunt. See the [`eks deploy`](https://github.com/gruntwork-io/kubergrunt#deploy) subcommand. - -## How do I create kubernetes resources when creating the cluster? - -You do not need to do anything extra since v12.1.0 of the module as long as the following conditions are met: - -- `manage_aws_auth = true` on the module (default) -- the kubernetes provider is correctly configured like in the [Usage Example](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/README.md#usage-example). Primarily the module's `cluster_id` output is used as input to the `aws_eks_cluster*` data sources. - -The `cluster_id` depends on a `data.http.wait_for_cluster` that polls the EKS cluster's endpoint until it is alive. This blocks initialisation of the kubernetes provider. - -## `aws_auth.tf: At 2:14: Unknown token: 2:14 IDENT` - -You are attempting to use a Terraform 0.12 module with Terraform 0.11. - -We highly recommend that you upgrade your EKS Terraform config to 0.12 to take advantage of new features in the module. - -Alternatively you can lock your module to a compatible version if you must stay with terraform 0.11: - -```hcl -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 4.0" - # ... -} -``` - -## How can I use Windows workers? - -To enable Windows support for your EKS cluster, you should apply some configs manually. See the [Enabling Windows Support (Windows/MacOS/Linux)](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support). - -Windows worker nodes requires additional cluster role (eks:kube-proxy-windows). If you are adding windows workers to existing cluster, you should apply config-map-aws-auth again. - -#### Example configuration - -Amazon EKS clusters must contain one or more Linux worker nodes to run core system pods that only run on Linux, such as coredns and the VPC resource controller. - -1. Build AWS EKS cluster with the next workers configuration (default Linux): - -```hcl -worker_groups = [ - { - name = "worker-group-linux" - instance_type = "m5.large" - platform = "linux" - asg_desired_capacity = 2 - }, - ] -``` - -2. Apply commands from https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support (use tab with name `Windows`) -3. Add one more worker group for Windows with required field `platform = "windows"` and update your cluster. Worker group example: - -```hcl -worker_groups = [ - { - name = "worker-group-linux" - instance_type = "m5.large" - platform = "linux" - asg_desired_capacity = 2 - }, - { - name = "worker-group-windows" - instance_type = "m5.large" - platform = "windows" - asg_desired_capacity = 1 - }, - ] -``` - -4. With `kubectl get nodes` you can see cluster with mixed (Linux/Windows) nodes support. - -## Worker nodes with labels do not join a 1.16+ cluster - -Kubelet restricts the allowed list of labels in the `kubernetes.io` namespace that can be applied to nodes starting in 1.16. - -Older configurations used labels like `kubernetes.io/lifecycle=spot` and this is no longer allowed. Use `node.kubernetes.io/lifecycle=spot` instead. - -Reference the `--node-labels` argument for your version of Kubenetes for the allowed prefixes. [Documentation for 1.16](https://v1-16.docs.kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) - -## What is the difference between `node_groups` and `worker_groups`? - -`node_groups` are [AWS-managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) (configures "Node Groups" that you can find on the EKS dashboard). This system is supposed to ease some of the lifecycle around upgrading nodes. Although they do not do this automatically and you still need to manually trigger the updates. - -`worker_groups` are [self-managed nodes](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) (provisions a typical "Autoscaling group" on EC2). It gives you full control over nodes in the cluster like using custom AMI for the nodes. As AWS says, "with worker groups the customer controls the data plane & AWS controls the control plane". - -Both can be used together in the same cluster. - -## I'm using both AWS-Managed node groups and Self-Managed worker groups and pods scheduled on a AWS Managed node groups are unable resolve DNS (even communication between pods) - -This happen because Core DNS can be scheduled on Self-Managed worker groups and by default, the terraform module doesn't create security group rules to ensure communication between pods schedulled on Self-Managed worker group and AWS-Managed node groups. - -You can set `var.worker_create_cluster_primary_security_group_rules` to `true` to create required rules. - -## Dedicated control plane subnets - -[AWS recommends](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html) to create dedicated subnets for EKS created network interfaces (control plane). The module fully supports this approach. To set up this, you must configure the module by adding additional `subnets` into workers default specification `workers_group_defaults` map or directly `subnets` definition in worker definition. - -```hcl -module "eks" { - source = "terraform-aws-modules/eks/aws" - - cluster_version = "1.21" - cluster_name = "my-cluster" - vpc_id = "vpc-1234556abcdef" - subnets = ["subnet-abcde123", "subnet-abcde456", "subnet-abcde789"] - - workers_group_defaults = { - subnets = ["subnet-xyz123", "subnet-xyz456", "subnet-xyz789"] - } - - worker_groups = [ - { - instance_type = "m4.large" - asg_max_size = 5 - }, - { - name = "worker-group-2" - subnets = ["subnet-qwer123"] - instance_type = "t3.medium" - asg_desired_capacity = 1 - public_ip = true - ebs_optimized = true - } - ] -} -``` \ No newline at end of file diff --git a/docs/iam-permissions.md b/docs/iam-permissions.md deleted file mode 100644 index e6a867a9ec..0000000000 --- a/docs/iam-permissions.md +++ /dev/null @@ -1,155 +0,0 @@ -# IAM Permissions - -Following IAM permissions are the minimum permissions needed for your IAM user or IAM role to create an EKS cluster. - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "autoscaling:AttachInstances", - "autoscaling:CreateAutoScalingGroup", - "autoscaling:CreateLaunchConfiguration", - "autoscaling:CreateOrUpdateTags", - "autoscaling:DeleteAutoScalingGroup", - "autoscaling:DeleteLaunchConfiguration", - "autoscaling:DeleteTags", - "autoscaling:Describe*", - "autoscaling:DetachInstances", - "autoscaling:SetDesiredCapacity", - "autoscaling:UpdateAutoScalingGroup", - "autoscaling:SuspendProcesses", - "ec2:AllocateAddress", - "ec2:AssignPrivateIpAddresses", - "ec2:Associate*", - "ec2:AttachInternetGateway", - "ec2:AttachNetworkInterface", - "ec2:AuthorizeSecurityGroupEgress", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateDefaultSubnet", - "ec2:CreateDhcpOptions", - "ec2:CreateEgressOnlyInternetGateway", - "ec2:CreateInternetGateway", - "ec2:CreateNatGateway", - "ec2:CreateNetworkInterface", - "ec2:CreateRoute", - "ec2:CreateRouteTable", - "ec2:CreateSecurityGroup", - "ec2:CreateSubnet", - "ec2:CreateTags", - "ec2:CreateVolume", - "ec2:CreateVpc", - "ec2:CreateVpcEndpoint", - "ec2:DeleteDhcpOptions", - "ec2:DeleteEgressOnlyInternetGateway", - "ec2:DeleteInternetGateway", - "ec2:DeleteNatGateway", - "ec2:DeleteNetworkInterface", - "ec2:DeleteRoute", - "ec2:DeleteRouteTable", - "ec2:DeleteSecurityGroup", - "ec2:DeleteSubnet", - "ec2:DeleteTags", - "ec2:DeleteVolume", - "ec2:DeleteVpc", - "ec2:DeleteVpnGateway", - "ec2:Describe*", - "ec2:DetachInternetGateway", - "ec2:DetachNetworkInterface", - "ec2:DetachVolume", - "ec2:Disassociate*", - "ec2:ModifySubnetAttribute", - "ec2:ModifyVpcAttribute", - "ec2:ModifyVpcEndpoint", - "ec2:ReleaseAddress", - "ec2:RevokeSecurityGroupEgress", - "ec2:RevokeSecurityGroupIngress", - "ec2:UpdateSecurityGroupRuleDescriptionsEgress", - "ec2:UpdateSecurityGroupRuleDescriptionsIngress", - "ec2:CreateLaunchTemplate", - "ec2:CreateLaunchTemplateVersion", - "ec2:DeleteLaunchTemplate", - "ec2:DeleteLaunchTemplateVersions", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeLaunchTemplateVersions", - "ec2:GetLaunchTemplateData", - "ec2:ModifyLaunchTemplate", - "ec2:RunInstances", - "eks:CreateCluster", - "eks:DeleteCluster", - "eks:DescribeCluster", - "eks:ListClusters", - "eks:UpdateClusterConfig", - "eks:UpdateClusterVersion", - "eks:DescribeUpdate", - "eks:TagResource", - "eks:UntagResource", - "eks:ListTagsForResource", - "eks:CreateFargateProfile", - "eks:DeleteFargateProfile", - "eks:DescribeFargateProfile", - "eks:ListFargateProfiles", - "eks:CreateNodegroup", - "eks:DeleteNodegroup", - "eks:DescribeNodegroup", - "eks:ListNodegroups", - "eks:UpdateNodegroupConfig", - "eks:UpdateNodegroupVersion", - "iam:AddRoleToInstanceProfile", - "iam:AttachRolePolicy", - "iam:CreateInstanceProfile", - "iam:CreateOpenIDConnectProvider", - "iam:CreateServiceLinkedRole", - "iam:CreatePolicy", - "iam:CreatePolicyVersion", - "iam:CreateRole", - "iam:DeleteInstanceProfile", - "iam:DeleteOpenIDConnectProvider", - "iam:DeletePolicy", - "iam:DeletePolicyVersion", - "iam:DeleteRole", - "iam:DeleteRolePolicy", - "iam:DeleteServiceLinkedRole", - "iam:DetachRolePolicy", - "iam:GetInstanceProfile", - "iam:GetOpenIDConnectProvider", - "iam:GetPolicy", - "iam:GetPolicyVersion", - "iam:GetRole", - "iam:GetRolePolicy", - "iam:List*", - "iam:PassRole", - "iam:PutRolePolicy", - "iam:RemoveRoleFromInstanceProfile", - "iam:TagOpenIDConnectProvider", - "iam:TagRole", - "iam:UntagRole", - "iam:TagPolicy", - "iam:TagInstanceProfile", - "iam:UpdateAssumeRolePolicy", - // Following permissions are needed if cluster_enabled_log_types is enabled - "logs:CreateLogGroup", - "logs:DescribeLogGroups", - "logs:DeleteLogGroup", - "logs:ListTagsLogGroup", - "logs:PutRetentionPolicy", - // Following permissions for working with secrets_encryption example - "kms:CreateAlias", - "kms:CreateGrant", - "kms:CreateKey", - "kms:DeleteAlias", - "kms:DescribeKey", - "kms:GetKeyPolicy", - "kms:GetKeyRotationStatus", - "kms:ListAliases", - "kms:ListResourceTags", - "kms:ScheduleKeyDeletion" - ], - "Resource": "*" - } - ] -} -``` diff --git a/docs/spot-instances.md b/docs/spot-instances.md deleted file mode 100644 index be89f52f87..0000000000 --- a/docs/spot-instances.md +++ /dev/null @@ -1,114 +0,0 @@ -# Using spot instances - -Spot instances usually cost around 30-70% less than an on-demand instance. So using them for your EKS workloads can save a lot of money but requires some special considerations as they could be terminated with only 2 minutes warning. - -You need to install a daemonset to catch the 2 minute warning before termination. This will ensure the node is gracefully drained before termination. You can install the [k8s-spot-termination-handler](https://github.com/kube-aws/kube-spot-termination-notice-handler) for this. There's a [Helm chart](https://github.com/helm/charts/tree/master/stable/k8s-spot-termination-handler): - -```shell -helm install stable/k8s-spot-termination-handler --namespace kube-system -``` - -In the following examples at least 1 worker group that uses on-demand instances is included. This worker group has an added node label that can be used in scheduling. This could be used to schedule any workload not suitable for spot instances but is important for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) as it might be end up unscheduled when spot instances are terminated. You can add this to the values of the [cluster-autoscaler helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler-chart): - -```yaml -nodeSelector: - kubernetes.io/lifecycle: normal -``` - -Notes: - -- The `spot_price` is set to the on-demand price so that the spot instances will run as long as they are the cheaper. -- It's best to have a broad range of instance types to ensure there's always some instances to run when prices fluctuate. -- There is an AWS blog article about this [here](https://aws.amazon.com/blogs/compute/run-your-kubernetes-workloads-on-amazon-ec2-spot-instances-with-amazon-eks/). -- Consider using [k8s-spot-rescheduler](https://github.com/pusher/k8s-spot-rescheduler) to move pods from on-demand to spot instances. - -## Using Launch Configuration - -Example worker group configuration that uses an ASG with launch configuration for each worker group: - -```hcl - worker_groups = [ - { - name = "on-demand-1" - instance_type = "m4.xlarge" - asg_max_size = 1 - kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=normal" - suspended_processes = ["AZRebalance"] - }, - { - name = "spot-1" - spot_price = "0.199" - instance_type = "c4.xlarge" - asg_max_size = 20 - kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" - suspended_processes = ["AZRebalance"] - }, - { - name = "spot-2" - spot_price = "0.20" - instance_type = "m4.xlarge" - asg_max_size = 20 - kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" - suspended_processes = ["AZRebalance"] - } - ] -``` - -## Using Launch Templates - -Launch Template support is a recent addition to both AWS and this module. It might not be as tried and tested but it's more suitable for spot instances as it allowed multiple instance types in the same worker group: - -```hcl - worker_groups = [ - { - name = "on-demand-1" - instance_type = "m4.xlarge" - asg_max_size = 10 - kubelet_extra_args = "--node-labels=spot=false" - suspended_processes = ["AZRebalance"] - } - ] - - - worker_groups_launch_template = [ - { - name = "spot-1" - override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] - spot_instance_pools = 4 - asg_max_size = 5 - asg_desired_capacity = 5 - kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" - public_ip = true - }, - ] -``` - -## Using Launch Templates With Both Spot and On Demand - -Example launch template to launch 2 on demand instances of type m5.large, and have the ability to scale up using spot instances and on demand instances. The `node.kubernetes.io/lifecycle` node label will be set to the value queried from the EC2 meta-data service: either "on-demand" or "spot". - -`on_demand_percentage_above_base_capacity` is set to 25 so 1 in 4 new nodes, when auto-scaling, will be on-demand instances. If not set, all new nodes will be spot instances. The on-demand instances will be the primary instance type (first in the array if they are not weighted). - -```hcl - worker_groups_launch_template = [{ - name = "mixed-demand-spot" - override_instance_types = ["m5.large", "m5a.large", "m4.large"] - root_encrypted = true - root_volume_size = 50 - - asg_min_size = 2 - asg_desired_capacity = 2 - on_demand_base_capacity = 3 - on_demand_percentage_above_base_capacity = 25 - asg_max_size = 20 - spot_instance_pools = 3 - - kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=`curl -s http://169.254.169.254/latest/meta-data/instance-life-cycle`" - }] -``` - -## Important Notes - -An issue with the cluster-autoscaler: https://github.com/kubernetes/autoscaler/issues/1133 - -AWS have released their own termination handler now: https://github.com/aws/aws-node-termination-handler diff --git a/examples/bottlerocket/README.md b/examples/bottlerocket/README.md deleted file mode 100644 index 88d9d4ee14..0000000000 --- a/examples/bottlerocket/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# AWS EKS cluster running Bottlerocket AMI - -Configuration in this directory creates EKS cluster with workers group running [AWS Bottlerocket OS](https://github.com/bottlerocket-os/bottlerocket) - -This is a minimalistic example which shows what knobs to turn to make Bottlerocket work. - -See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami-bottlerocket.html) for more details. - -## Usage - -To run this example you need to execute: - -```bash -$ terraform init -$ terraform plan -$ terraform apply -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | -| [local](#requirement\_local) | >= 1.4 | -| [random](#requirement\_random) | >= 2.1 | -| [tls](#requirement\_tls) | >= 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [random](#provider\_random) | >= 2.1 | -| [tls](#provider\_tls) | >= 2.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | ../.. | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_iam_role_policy_attachment.ssm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_key_pair.nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | -| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | -| [tls_private_key.nodes](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | -| [aws_ami.bottlerocket_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | -| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | - -## Inputs - -No inputs. - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | -| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | -| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | -| [node\_groups](#output\_node\_groups) | Outputs from node groups | - diff --git a/examples/bottlerocket/main.tf b/examples/bottlerocket/main.tf deleted file mode 100644 index bffa83238b..0000000000 --- a/examples/bottlerocket/main.tf +++ /dev/null @@ -1,159 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - name = "bottlerocket-${random_string.suffix.result}" - cluster_version = "1.20" - region = "eu-west-1" -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_version = local.cluster_version - - vpc_id = module.vpc.vpc_id - subnets = [module.vpc.private_subnets[0], module.vpc.public_subnets[1]] - fargate_subnets = [module.vpc.private_subnets[2]] - - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - - write_kubeconfig = false - manage_aws_auth = true - - worker_groups_launch_template = [ - { - name = "bottlerocket-nodes" - ami_id = data.aws_ami.bottlerocket_ami.id - instance_type = "t3a.small" - asg_desired_capacity = 2 - key_name = aws_key_pair.nodes.key_name - - # Since we are using default VPC there is no NAT gateway so we need to - # attach public ip to nodes so they can reach k8s API server - # do not repeat this at home (i.e. production) - public_ip = true - - # This section overrides default userdata template to pass bottlerocket - # specific user data - userdata_template_file = "${path.module}/userdata.toml" - # we are using this section to pass additional arguments for - # userdata template rendering - userdata_template_extra_args = { - enable_admin_container = false - enable_control_container = true - aws_region = data.aws_region.current.name - } - # example of k8s/kubelet configuration via additional_userdata - additional_userdata = < [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | -| [local](#requirement\_local) | >= 1.4 | -| [random](#requirement\_random) | >= 2.1 | +| [aws](#requirement\_aws) | >= 3.64 | +| [null](#requirement\_null) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [random](#provider\_random) | >= 2.1 | +| [aws](#provider\_aws) | >= 3.64 | +| [null](#provider\_null) | >= 3.0 | ## Modules | Name | Source | Version | |------|--------|---------| | [disabled\_eks](#module\_disabled\_eks) | ../.. | n/a | -| [disabled\_fargate](#module\_disabled\_fargate) | ../../modules/fargate | n/a | -| [disabled\_node\_groups](#module\_disabled\_node\_groups) | ../../modules/node_groups | n/a | +| [disabled\_eks\_managed\_node\_group](#module\_disabled\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | +| [disabled\_fargate\_profile](#module\_disabled\_fargate\_profile) | ../../modules/fargate-profile | n/a | +| [disabled\_self\_managed\_node\_group](#module\_disabled\_self\_managed\_node\_group) | ../../modules/self-managed-node-group | n/a | | [eks](#module\_eks) | ../.. | n/a | +| [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | +| [fargate\_profile](#module\_fargate\_profile) | ../../modules/fargate-profile | n/a | +| [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ../../modules/self-managed-node-group | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources | Name | Type | |------|------| -| [aws_security_group.all_worker_mgmt](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_security_group.worker_group_mgmt_one](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_security_group.worker_group_mgmt_two](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | -| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [null_resource.patch](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | ## Inputs @@ -66,9 +75,25 @@ No inputs. | Name | Description | |------|-------------| -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | -| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | -| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | -| [node\_groups](#output\_node\_groups) | Outputs from node groups | +| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | +| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | +| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | +| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | +| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | +| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | +| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | +| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 5cfcab668b..ff32473737 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -3,9 +3,15 @@ provider "aws" { } locals { - name = "complete-${random_string.suffix.result}" - cluster_version = "1.20" + name = "ex-${replace(basename(path.cwd), "_", "-")}" + cluster_version = "1.21" region = "eu-west-1" + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } } ################################################################################ @@ -15,87 +21,102 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - - vpc_id = module.vpc.vpc_id - subnets = [module.vpc.private_subnets[0], module.vpc.public_subnets[1]] - fargate_subnets = [module.vpc.private_subnets[2]] - + cluster_name = local.name + cluster_version = local.cluster_version cluster_endpoint_private_access = true cluster_endpoint_public_access = true + cluster_addons = { + coredns = { + resolve_conflicts = "OVERWRITE" + } + kube-proxy = {} + vpc-cni = { + resolve_conflicts = "OVERWRITE" + } + } + cluster_encryption_config = [{ + provider_key_arn = aws_kms_key.eks.arn + resources = ["secrets"] + }] - worker_additional_security_group_ids = [aws_security_group.all_worker_mgmt.id] - - # Worker groups (using Launch Configurations) - worker_groups = [ - { - name = "worker-group-1" - instance_type = "t3.small" - additional_userdata = "echo foo bar" - asg_desired_capacity = 2 - additional_security_group_ids = [aws_security_group.worker_group_mgmt_one.id] - }, - { - name = "worker-group-2" - instance_type = "t3.medium" - additional_userdata = "echo foo bar" - additional_security_group_ids = [aws_security_group.worker_group_mgmt_two.id] - asg_desired_capacity = 1 - }, - ] + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets - # Worker groups (using Launch Templates) - worker_groups_launch_template = [ - { - name = "spot-1" - override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] - spot_instance_pools = 4 - asg_max_size = 5 - asg_desired_capacity = 5 - kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" - public_ip = true - }, - ] + enable_irsa = true - # Managed Node Groups - node_groups_defaults = { - ami_type = "AL2_x86_64" - disk_size = 50 + # Self Managed Node Group(s) + self_managed_node_group_defaults = { + vpc_security_group_ids = [aws_security_group.additional.id] + iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"] } - node_groups = { - example = { - desired_capacity = 1 - max_capacity = 10 - min_capacity = 1 + self_managed_node_groups = { + spot = { + instance_type = "m5.large" + instance_market_options = { + market_type = "spot" + } + + pre_bootstrap_user_data = <<-EOT + echo "foo" + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + post_bootstrap_user_data = <<-EOT + cd /tmp + sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm + sudo systemctl enable amazon-ssm-agent + sudo systemctl start amazon-ssm-agent + EOT + } + } + + # EKS Managed Node Group(s) + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] + vpc_security_group_ids = [aws_security_group.additional.id] + } + + eks_managed_node_groups = { + blue = {} + green = { + min_size = 1 + max_size = 10 + desired_size = 1 instance_types = ["t3.large"] capacity_type = "SPOT" - k8s_labels = { + labels = { Environment = "test" GithubRepo = "terraform-aws-eks" GithubOrg = "terraform-aws-modules" } - additional_tags = { - ExtraTag = "example" - } - taints = [ - { + + taints = { + dedicated = { key = "dedicated" value = "gpuGroup" effect = "NO_SCHEDULE" } - ] + } + update_config = { max_unavailable_percentage = 50 # or set `max_unavailable` } + + tags = { + ExtraTag = "example" + } } } - # Fargate + # Fargate Profile(s) fargate_profiles = { default = { name = "default" @@ -122,38 +143,59 @@ module "eks" { } } - # AWS Auth (kubernetes_config_map) - map_roles = [ - { - rolearn = "arn:aws:iam::66666666666:role/role1" - username = "role1" - groups = ["system:masters"] - }, - ] + tags = local.tags +} - map_users = [ - { - userarn = "arn:aws:iam::66666666666:user/user1" - username = "user1" - groups = ["system:masters"] - }, - { - userarn = "arn:aws:iam::66666666666:user/user2" - username = "user2" - groups = ["system:masters"] - }, - ] +################################################################################ +# Sub-Module Usage on Existing/Separate Cluster +################################################################################ + +module "eks_managed_node_group" { + source = "../../modules/eks-managed-node-group" + + name = "separate-eks-mng" + cluster_name = module.eks.cluster_id + cluster_version = local.cluster_version + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + tags = merge(local.tags, { Separate = "eks-managed-node-group" }) +} - map_accounts = [ - "777777777777", - "888888888888", +module "self_managed_node_group" { + source = "../../modules/self-managed-node-group" + + name = "separate-self-mng" + cluster_name = module.eks.cluster_id + cluster_version = local.cluster_version + cluster_endpoint = module.eks.cluster_endpoint + cluster_auth_base64 = module.eks.cluster_certificate_authority_data + + instance_type = "m5.large" + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + vpc_security_group_ids = [ + module.eks.cluster_primary_security_group_id, + module.eks.cluster_security_group_id, ] - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } + tags = merge(local.tags, { Separate = "self-managed-node-group" }) +} + +module "fargate_profile" { + source = "../../modules/fargate-profile" + + name = "separate-fargate-profile" + cluster_name = module.eks.cluster_id + + subnet_ids = module.vpc.private_subnets + selectors = [{ + namespace = "kube-system" + }] + + tags = merge(local.tags, { Separate = "fargate-profile" }) } ################################################################################ @@ -163,128 +205,159 @@ module "eks" { module "disabled_eks" { source = "../.." - create_eks = false + create = false +} + +module "disabled_fargate_profile" { + source = "../../modules/fargate-profile" + + create = false } -module "disabled_fargate" { - source = "../../modules/fargate" +module "disabled_eks_managed_node_group" { + source = "../../modules/eks-managed-node-group" - create_fargate_pod_execution_role = false + create = false } -module "disabled_node_groups" { - source = "../../modules/node_groups" +module "disabled_self_managed_node_group" { + source = "../../modules/self-managed-node-group" - create_eks = false + create = false } ################################################################################ -# Kubernetes provider configuration +# aws-auth configmap +# Only EKS managed node groups automatically add roles to aws-auth configmap +# so we need to ensure fargate profiles and self-managed node roles are added ################################################################################ -data "aws_eks_cluster" "cluster" { +data "aws_eks_cluster_auth" "this" { name = module.eks.cluster_id } -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id +locals { + kubeconfig = yamlencode({ + apiVersion = "v1" + kind = "Config" + current-context = "terraform" + clusters = [{ + name = module.eks.cluster_id + cluster = { + certificate-authority-data = module.eks.cluster_certificate_authority_data + server = module.eks.cluster_endpoint + } + }] + contexts = [{ + name = "terraform" + context = { + cluster = module.eks.cluster_id + user = "terraform" + } + }] + users = [{ + name = "terraform" + user = { + token = data.aws_eks_cluster_auth.this.token + } + }] + }) + + # we have to combine the configmap created by the eks module with the externally created node group/profile sub-modules + aws_auth_configmap_yaml = <<-EOT + ${chomp(module.eks.aws_auth_configmap_yaml)} + - rolearn: ${module.eks_managed_node_group.iam_role_arn} + username: system:node:{{EC2PrivateDNSName}} + groups: + - system:bootstrappers + - system:nodes + - rolearn: ${module.self_managed_node_group.iam_role_arn} + username: system:node:{{EC2PrivateDNSName}} + groups: + - system:bootstrappers + - system:nodes + - rolearn: ${module.fargate_profile.fargate_profile_arn} + username: system:node:{{SessionName}} + groups: + - system:bootstrappers + - system:nodes + - system:node-proxier + EOT } -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token +resource "null_resource" "patch" { + triggers = { + kubeconfig = base64encode(local.kubeconfig) + cmd_patch = "kubectl patch configmap/aws-auth --patch \"${local.aws_auth_configmap_yaml}\" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode)" + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = self.triggers.kubeconfig + } + command = self.triggers.cmd_patch + } } ################################################################################ -# Additional security groups for workers +# Supporting resources ################################################################################ -resource "aws_security_group" "worker_group_mgmt_one" { - name_prefix = "worker_group_mgmt_one" - vpc_id = module.vpc.vpc_id +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" + name = local.name + cidr = "10.0.0.0/16" - cidr_blocks = [ - "10.0.0.0/8", - ] - } -} + azs = ["${local.region}a", "${local.region}b", "${local.region}c"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] -resource "aws_security_group" "worker_group_mgmt_two" { - name_prefix = "worker_group_mgmt_two" - vpc_id = module.vpc.vpc_id + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" + enable_flow_log = true + create_flow_log_cloudwatch_iam_role = true + create_flow_log_cloudwatch_log_group = true - cidr_blocks = [ - "192.168.0.0/16", - ] + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = 1 } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags } -resource "aws_security_group" "all_worker_mgmt" { - name_prefix = "all_worker_management" +resource "aws_security_group" "additional" { + name_prefix = "${local.name}-additional" vpc_id = module.vpc.vpc_id ingress { from_port = 22 to_port = 22 protocol = "tcp" - cidr_blocks = [ "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", ] } -} - -################################################################################ -# Supporting resources -################################################################################ - -data "aws_availability_zones" "available" { -} -resource "random_string" "suffix" { - length = 8 - special = false + tags = local.tags } -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } +resource "aws_kms_key" "eks" { + description = "EKS Secret Encryption Key" + deletion_window_in_days = 7 + enable_key_rotation = true - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } + tags = local.tags } diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index 10a3a96604..8a9153c099 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -1,24 +1,148 @@ +################################################################################ +# Cluster +################################################################################ + +output "cluster_arn" { + description = "The Amazon Resource Name (ARN) of the cluster" + value = module.eks.cluster_arn +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded certificate data required to communicate with the cluster" + value = module.eks.cluster_certificate_authority_data +} + output "cluster_endpoint" { - description = "Endpoint for EKS control plane." + description = "Endpoint for your Kubernetes API server" value = module.eks.cluster_endpoint } +output "cluster_id" { + description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_id +} + +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_oidc_issuer_url +} + +output "cluster_platform_version" { + description = "Platform version for the cluster" + value = module.eks.cluster_platform_version +} + +output "cluster_status" { + description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" + value = module.eks.cluster_status +} + output "cluster_security_group_id" { - description = "Security group ids attached to the cluster control plane." + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" value = module.eks.cluster_security_group_id } -output "kubectl_config" { - description = "kubectl config as generated by the module." - value = module.eks.kubeconfig +################################################################################ +# Security Group +################################################################################ + +output "cluster_security_group_arn" { + description = "Amazon Resource Name (ARN) of the cluster security group" + value = module.eks.cluster_security_group_arn +} + +################################################################################ +# IRSA +################################################################################ + +output "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + value = module.eks.oidc_provider_arn +} + +################################################################################ +# IAM Role +################################################################################ + +output "cluster_iam_role_name" { + description = "IAM role name of the EKS cluster" + value = module.eks.cluster_iam_role_name +} + +output "cluster_iam_role_arn" { + description = "IAM role ARN of the EKS cluster" + value = module.eks.cluster_iam_role_arn } -output "config_map_aws_auth" { - description = "A kubernetes configuration to authenticate to this EKS cluster." - value = module.eks.config_map_aws_auth +output "cluster_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.cluster_iam_role_unique_id } -output "node_groups" { - description = "Outputs from node groups" - value = module.eks.node_groups +################################################################################ +# EKS Addons +################################################################################ + +output "cluster_addons" { + description = "Map of attribute maps for all EKS cluster addons enabled" + value = module.eks.cluster_addons +} + +################################################################################ +# EKS Identity Provider +################################################################################ + +output "cluster_identity_providers" { + description = "Map of attribute maps for all EKS identity providers enabled" + value = module.eks.cluster_identity_providers +} + +################################################################################ +# CloudWatch Log Group +################################################################################ + +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_name +} + +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_arn +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.eks.fargate_profiles +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +output "eks_managed_node_groups" { + description = "Map of attribute maps for all EKS managed node groups created" + value = module.eks.eks_managed_node_groups +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +output "self_managed_node_groups" { + description = "Map of attribute maps for all self managed node groups created" + value = module.eks.self_managed_node_groups +} + +################################################################################ +# Additional +################################################################################ + +output "aws_auth_configmap_yaml" { + description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" + value = module.eks.aws_auth_configmap_yaml } diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index 8e2b837984..adfd0180d4 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -4,19 +4,11 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.56" + version = ">= 3.64" } - local = { - source = "hashicorp/local" - version = ">= 1.4" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 1.11.1" - } - random = { - source = "hashicorp/random" - version = ">= 2.1" + null = { + source = "hashicorp/null" + version = ">= 3.0" } } } diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md new file mode 100644 index 0000000000..b23c13d3f1 --- /dev/null +++ b/examples/eks_managed_node_group/README.md @@ -0,0 +1,95 @@ +# EKS Managed Node Group Example + +Configuration in this directory creates an AWS EKS cluster with various EKS Managed Node Groups demonstrating the various methods of configuring/customizing: + +- A default, "out of the box" EKS managed node group as supplied by AWS EKS +- A default, "out of the box" Bottlerocket EKS managed node group as supplied by AWS EKS +- A Bottlerocket EKS managed node group that supplies additional bootstrap settings +- A Bottlerocket EKS managed node group that demonstrates many of the configuration/customizations offered by the `eks-managed-node-group` sub-module for the Bottlerocket OS +- An EKS managed node group created from a launch template created outside of the module +- An EKS managed node group that utilizes a custom AMI that is an EKS optimized AMI derivative +- An EKS managed node group that demonstrates nearly all of the configurations/customizations offered by the `eks-managed-node-group` sub-module + +See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for further details. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.64 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.64 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | n/a | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_kms_key.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_launch_template.external](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [null_resource.patch](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | +| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | +| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | +| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | +| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | +| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | +| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | +| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | +| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | + diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf new file mode 100644 index 0000000000..fb801c1eb1 --- /dev/null +++ b/examples/eks_managed_node_group/main.tf @@ -0,0 +1,490 @@ +provider "aws" { + region = local.region +} + +locals { + name = "ex-${replace(basename(path.cwd), "_", "-")}" + cluster_version = "1.21" + region = "eu-west-1" + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +data "aws_caller_identity" "current" {} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_service_ipv4_cidr = "172.16.0.0/16" + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + cluster_addons = { + coredns = { + resolve_conflicts = "OVERWRITE" + } + kube-proxy = {} + vpc-cni = { + resolve_conflicts = "OVERWRITE" + } + } + + cluster_encryption_config = [{ + provider_key_arn = aws_kms_key.eks.arn + resources = ["secrets"] + }] + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + enable_irsa = true + + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] + } + + eks_managed_node_groups = { + # Default node group - as provided by AWS EKS + default_node_group = {} + + # Default node group - as provided by AWS EKS using Bottlerocket + bottlerocket_default = { + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + } + + # Adds to the AWS provided user data + bottlerocket_add = { + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + + # this will get added to what AWS provides + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + EOT + } + + # Custom AMI, using module provided bootstrap data + bottlerocket_custom = { + # Current bottlerocket AMI + ami_id = "ami-0ff61e0bcfc81dc94" + platform = "bottlerocket" + + # use module user data template to boostrap + enable_bootstrap_user_data = true + # this will get added to the template + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + + [settings.kubernetes.node-labels] + "label1" = "foo" + "label2" = "bar" + + [settings.kubernetes.node-taints] + "dedicated" = "experimental:PreferNoSchedule" + "special" = "true:NoSchedule" + EOT + } + + # Use existing/external launch template + external_lt = { + create_launch_template = false + launch_template_name = aws_launch_template.external.name + launch_template_version = aws_launch_template.external.default_version + } + + # Use a custom AMI + custom_ami = { + # Current default AMI used by managed node groups - pseudo "custom" + ami_id = "ami-0caf35bc73450c396" + + # This will ensure the boostrap user data is used to join the node + # By default, EKS managed node groups will not append bootstrap script; + # this adds it back in using the default template provided by the module + # Note: this assumes the AMI provided is an EKS optimized AMI derivative + enable_bootstrap_user_data = true + } + + # Complete + complete = { + name = "complete-eks-mng" + use_name_prefix = false + + subnet_ids = module.vpc.private_subnets + + min_size = 1 + max_size = 7 + desired_size = 1 + + ami_id = "ami-0caf35bc73450c396" + enable_bootstrap_user_data = true + bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" + + pre_bootstrap_user_data = <<-EOT + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + EOT + + post_bootstrap_user_data = <<-EOT + echo "you are free little kubelet!" + EOT + + capacity_type = "SPOT" + disk_size = 256 + force_update_version = true + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large", "m3.large", "m4.large"] + labels = { + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + taints = [ + { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + ] + + update_config = { + max_unavailable_percentage = 50 # or set `max_unavailable` + } + + description = "EKS managed node group example launch template" + + ebs_optimized = true + vpc_security_group_ids = [aws_security_group.additional.id] + disable_api_termination = false + enable_monitoring = true + + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = 75 + volume_type = "gp3" + iops = 3000 + throughput = 150 + encrypted = true + kms_key_id = aws_kms_key.ebs.arn + delete_on_termination = true + } + } + } + + metadata_options = { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 2 + } + + create_iam_role = true + iam_role_name = "eks-managed-node-group-complete-example" + iam_role_use_name_prefix = false + iam_role_description = "EKS managed node group complete example role" + iam_role_tags = { + Purpose = "Protector of the kubelet" + } + iam_role_additional_policies = [ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + ] + + create_security_group = true + security_group_name = "eks-managed-node-group-complete-example" + security_group_use_name_prefix = false + security_group_description = "EKS managed node group complete example security group" + security_group_rules = { + phoneOut = { + description = "Hello CloudFlare" + protocol = "udp" + from_port = 53 + to_port = 53 + type = "egress" + cidr_blocks = ["1.1.1.1/32"] + } + phoneHome = { + description = "Hello cluster" + protocol = "udp" + from_port = 53 + to_port = 53 + type = "egress" + source_cluster_security_group = true # bit of reflection lookup + } + } + security_group_tags = { + Purpose = "Protector of the kubelet" + } + + tags = { + ExtraTag = "EKS managed node group complete example" + } + } + } + + tags = local.tags +} + +################################################################################ +# aws-auth configmap +# Only EKS managed node groups automatically add roles to aws-auth configmap +# so we need to ensure fargate profiles and self-managed node roles are added +################################################################################ + +data "aws_eks_cluster_auth" "this" { + name = module.eks.cluster_id +} + +locals { + kubeconfig = yamlencode({ + apiVersion = "v1" + kind = "Config" + current-context = "terraform" + clusters = [{ + name = module.eks.cluster_id + cluster = { + certificate-authority-data = module.eks.cluster_certificate_authority_data + server = module.eks.cluster_endpoint + } + }] + contexts = [{ + name = "terraform" + context = { + cluster = module.eks.cluster_id + user = "terraform" + } + }] + users = [{ + name = "terraform" + user = { + token = data.aws_eks_cluster_auth.this.token + } + }] + }) +} + +resource "null_resource" "patch" { + triggers = { + kubeconfig = base64encode(local.kubeconfig) + cmd_patch = "kubectl patch configmap/aws-auth --patch \"${module.eks.aws_auth_configmap_yaml}\" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode)" + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = self.triggers.kubeconfig + } + command = self.triggers.cmd_patch + } +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + + azs = ["${local.region}a", "${local.region}b", "${local.region}c"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + enable_flow_log = true + create_flow_log_cloudwatch_iam_role = true + create_flow_log_cloudwatch_log_group = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} + +resource "aws_security_group" "additional" { + name_prefix = "${local.name}-additional" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + ] + } + + tags = local.tags +} + +resource "aws_kms_key" "eks" { + description = "EKS Secret Encryption Key" + deletion_window_in_days = 7 + enable_key_rotation = true + + tags = local.tags +} + +resource "aws_kms_key" "ebs" { + description = "Customer managed key to encrypt EKS managed node group volumes" + deletion_window_in_days = 7 + policy = data.aws_iam_policy_document.ebs.json +} + +# This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes +data "aws_iam_policy_document" "ebs" { + # Copy of default KMS policy that lets you manage it + statement { + sid = "Enable IAM User Permissions" + actions = ["kms:*"] + resources = ["*"] + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + } + } + + # Required for EKS + statement { + sid = "Allow service-linked role use of the CMK" + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs + ] + } + } + + statement { + sid = "Allow attachment of persistent resources" + actions = ["kms:CreateGrant"] + resources = ["*"] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs + ] + } + + condition { + test = "Bool" + variable = "kms:GrantIsForAWSResource" + values = ["true"] + } + } +} + +# This is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) +# there are several more options one could set but you probably dont need to modify them +# you can take the default and add your custom AMI and/or custom tags +# +# Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, +# then the default user-data for bootstrapping a cluster is merged in the copy. + +resource "aws_launch_template" "external" { + name_prefix = "external-eks-ex-" + description = "EKS managed node group external launch template" + update_default_version = true + + block_device_mappings { + device_name = "/dev/xvda" + + ebs { + volume_size = 100 + volume_type = "gp2" + delete_on_termination = true + } + } + + monitoring { + enabled = true + } + + network_interfaces { + associate_public_ip_address = false + delete_on_termination = true + } + + # if you want to use a custom AMI + # image_id = var.ami_id + + # If you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then + # you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 + # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) + # user_data = base64encode(data.template_file.launch_template_userdata.rendered) + + tag_specifications { + resource_type = "instance" + + tags = { + CustomTag = "Instance custom tag" + } + } + + tag_specifications { + resource_type = "volume" + + tags = { + CustomTag = "Volume custom tag" + } + } + + tag_specifications { + resource_type = "network-interface" + + tags = { + CustomTag = "EKS example" + } + } + + tags = { + CustomTag = "Launch template custom tag" + } + + lifecycle { + create_before_destroy = true + } +} diff --git a/examples/eks_managed_node_group/outputs.tf b/examples/eks_managed_node_group/outputs.tf new file mode 100644 index 0000000000..3e9620157b --- /dev/null +++ b/examples/eks_managed_node_group/outputs.tf @@ -0,0 +1,167 @@ +################################################################################ +# Cluster +################################################################################ + +output "cluster_arn" { + description = "The Amazon Resource Name (ARN) of the cluster" + value = module.eks.cluster_arn +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded certificate data required to communicate with the cluster" + value = module.eks.cluster_certificate_authority_data +} + +output "cluster_endpoint" { + description = "Endpoint for your Kubernetes API server" + value = module.eks.cluster_endpoint +} + +output "cluster_id" { + description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_id +} + +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_oidc_issuer_url +} + +output "cluster_platform_version" { + description = "Platform version for the cluster" + value = module.eks.cluster_platform_version +} + +output "cluster_status" { + description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" + value = module.eks.cluster_status +} + +output "cluster_primary_security_group_id" { + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = module.eks.cluster_primary_security_group_id +} + +################################################################################ +# Security Group +################################################################################ + +output "cluster_security_group_arn" { + description = "Amazon Resource Name (ARN) of the cluster security group" + value = module.eks.cluster_security_group_arn +} + +output "cluster_security_group_id" { + description = "ID of the cluster security group" + value = module.eks.cluster_security_group_id +} + +################################################################################ +# Node Security Group +################################################################################ + +output "node_security_group_arn" { + description = "Amazon Resource Name (ARN) of the node shared security group" + value = module.eks.node_security_group_arn +} + +output "node_security_group_id" { + description = "ID of the node shared security group" + value = module.eks.node_security_group_id +} + +################################################################################ +# IRSA +################################################################################ + +output "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + value = module.eks.oidc_provider_arn +} + +################################################################################ +# IAM Role +################################################################################ + +output "cluster_iam_role_name" { + description = "IAM role name of the EKS cluster" + value = module.eks.cluster_iam_role_name +} + +output "cluster_iam_role_arn" { + description = "IAM role ARN of the EKS cluster" + value = module.eks.cluster_iam_role_arn +} + +output "cluster_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.cluster_iam_role_unique_id +} + +################################################################################ +# EKS Addons +################################################################################ + +output "cluster_addons" { + description = "Map of attribute maps for all EKS cluster addons enabled" + value = module.eks.cluster_addons +} + +################################################################################ +# EKS Identity Provider +################################################################################ + +output "cluster_identity_providers" { + description = "Map of attribute maps for all EKS identity providers enabled" + value = module.eks.cluster_identity_providers +} + +################################################################################ +# CloudWatch Log Group +################################################################################ + +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_name +} + +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_arn +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.eks.fargate_profiles +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +output "eks_managed_node_groups" { + description = "Map of attribute maps for all EKS managed node groups created" + value = module.eks.eks_managed_node_groups +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +output "self_managed_node_groups" { + description = "Map of attribute maps for all self managed node groups created" + value = module.eks.self_managed_node_groups +} + +################################################################################ +# Additional +################################################################################ + +output "aws_auth_configmap_yaml" { + description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" + value = module.eks.aws_auth_configmap_yaml +} diff --git a/examples/bottlerocket/variables.tf b/examples/eks_managed_node_group/variables.tf similarity index 100% rename from examples/bottlerocket/variables.tf rename to examples/eks_managed_node_group/variables.tf diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf new file mode 100644 index 0000000000..adfd0180d4 --- /dev/null +++ b/examples/eks_managed_node_group/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.64" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } +} diff --git a/examples/fargate/README.md b/examples/fargate/README.md deleted file mode 100644 index 1228f5c4c1..0000000000 --- a/examples/fargate/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# AWS EKS Cluster with Fargate profiles - -Configuration in this directory creates EKS cluster with Fargate profiles in two different ways: - -- Using a root module, where EKS Cluster and Fargate profiles should be created at once. This is the default behaviour for most users. -- Using `modules/fargate` submodule where Fargate profiles should be attached to the existing EKS Cluster. - -## Usage - -To run this example you need to execute: - -```bash -$ terraform init -$ terraform plan -$ terraform apply -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | -| [local](#requirement\_local) | >= 1.4 | -| [random](#requirement\_random) | >= 2.1 | -| [tls](#requirement\_tls) | >= 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [random](#provider\_random) | >= 2.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | ../.. | n/a | -| [fargate\_profile\_existing\_cluster](#module\_fargate\_profile\_existing\_cluster) | ../../modules/fargate | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | - -## Resources - -| Name | Type | -|------|------| -| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | -| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | - -## Inputs - -No inputs. - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | -| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | -| [fargate\_profile\_arns](#output\_fargate\_profile\_arns) | Outputs from node groups | -| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | - diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf deleted file mode 100644 index 09e2e57e2d..0000000000 --- a/examples/fargate/main.tf +++ /dev/null @@ -1,235 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - name = "fargate-${random_string.suffix.result}" - cluster_version = "1.20" - region = "eu-west-1" -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_version = local.cluster_version - - vpc_id = module.vpc.vpc_id - subnets = [module.vpc.private_subnets[0], module.vpc.public_subnets[1]] - fargate_subnets = [module.vpc.private_subnets[2]] - - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - - # You require a node group to schedule coredns which is critical for running correctly internal DNS. - # If you want to use only fargate you must follow docs `(Optional) Update CoreDNS` - # available under https://docs.aws.amazon.com/eks/latest/userguide/fargate-getting-started.html - node_groups = { - example = { - desired_capacity = 1 - - instance_types = ["t3.large"] - k8s_labels = { - Example = "managed_node_groups" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - additional_tags = { - ExtraTag = "example" - } - update_config = { - max_unavailable_percentage = 50 # or set `max_unavailable` - } - } - } - - fargate_profiles = { - default = { - name = "default" - selectors = [ - { - namespace = "kube-system" - labels = { - k8s-app = "kube-dns" - } - }, - { - namespace = "default" - labels = { - WorkerType = "fargate" - } - } - ] - - tags = { - Owner = "default" - } - - timeouts = { - create = "20m" - delete = "20m" - } - } - - secondary = { - name = "secondary" - selectors = [ - { - namespace = "default" - labels = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - } - ] - - # Using specific subnets instead of the ones configured in EKS (`subnets` and `fargate_subnets`) - subnets = [module.vpc.private_subnets[1]] - - tags = { - Owner = "secondary" - } - } - } - - manage_aws_auth = false - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - - -############################################## -# Calling submodule with existing EKS cluster -############################################## - -module "fargate_profile_existing_cluster" { - source = "../../modules/fargate" - - cluster_name = module.eks.cluster_id - subnets = [module.vpc.private_subnets[0], module.vpc.private_subnets[2]] - - fargate_profiles = { - profile1 = { - name = "profile1" - selectors = [ - { - namespace = "kube-system" - labels = { - k8s-app = "kube-dns" - } - }, - { - namespace = "profile" - labels = { - WorkerType = "fargate" - } - } - ] - - tags = { - Owner = "profile1" - submodule = "true" - } - } - - profile2 = { - name = "profile2" - selectors = [ - { - namespace = "default" - labels = { - Fargate = "profile2" - } - } - ] - - # Using specific subnets instead of the ones configured in EKS (`subnets` and `fargate_subnets`) - subnets = [module.vpc.private_subnets[0]] - - tags = { - Owner = "profile2" - submodule = "true" - } - - timeouts = { - delete = "20m" - } - } - } - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# Kubernetes provider configuration -################################################################################ - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} - -################################################################################ -# Supporting Resources -################################################################################ - -data "aws_availability_zones" "available" { -} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - diff --git a/examples/fargate/outputs.tf b/examples/fargate/outputs.tf deleted file mode 100644 index b7f23eeaf2..0000000000 --- a/examples/fargate/outputs.tf +++ /dev/null @@ -1,24 +0,0 @@ -output "cluster_endpoint" { - description = "Endpoint for EKS control plane." - value = module.eks.cluster_endpoint -} - -output "cluster_security_group_id" { - description = "Security group ids attached to the cluster control plane." - value = module.eks.cluster_security_group_id -} - -output "kubectl_config" { - description = "kubectl config as generated by the module." - value = module.eks.kubeconfig -} - -output "config_map_aws_auth" { - description = "A kubernetes configuration to authenticate to this EKS cluster." - value = module.eks.config_map_aws_auth -} - -output "fargate_profile_arns" { - description = "Outputs from node groups" - value = module.eks.fargate_profile_arns -} diff --git a/examples/fargate/versions.tf b/examples/fargate/versions.tf deleted file mode 100644 index cb5115c487..0000000000 --- a/examples/fargate/versions.tf +++ /dev/null @@ -1,26 +0,0 @@ -terraform { - required_version = ">= 0.13.1" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.56" - } - local = { - source = "hashicorp/local" - version = ">= 1.4" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 1.11.1" - } - random = { - source = "hashicorp/random" - version = ">= 2.1" - } - tls = { - source = "hashicorp/tls" - version = ">= 2.0" - } - } -} diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md new file mode 100644 index 0000000000..9ff7b03a4c --- /dev/null +++ b/examples/fargate_profile/README.md @@ -0,0 +1,76 @@ +# AWS EKS Cluster with Fargate profiles + +Configuration in this directory creates an AWS EKS cluster utilizing Fargate profiles. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.64 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.64 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | n/a | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | +| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | +| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | +| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | +| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | +| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | +| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | +| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | +| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | + diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf new file mode 100644 index 0000000000..61fbb43671 --- /dev/null +++ b/examples/fargate_profile/main.tf @@ -0,0 +1,164 @@ +provider "aws" { + region = local.region +} + +locals { + name = "ex-${replace(basename(path.cwd), "_", "-")}" + cluster_version = "1.21" + region = "eu-west-1" + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + cluster_addons = { + # Note: https://docs.aws.amazon.com/eks/latest/userguide/fargate-getting-started.html#fargate-gs-coredns + coredns = { + resolve_conflicts = "OVERWRITE" + } + kube-proxy = {} + vpc-cni = { + resolve_conflicts = "OVERWRITE" + } + } + + cluster_encryption_config = [{ + provider_key_arn = aws_kms_key.eks.arn + resources = ["secrets"] + }] + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + enable_irsa = true + + # You require a node group to schedule coredns which is critical for running correctly internal DNS. + # If you want to use only fargate you must follow docs `(Optional) Update CoreDNS` + # available under https://docs.aws.amazon.com/eks/latest/userguide/fargate-getting-started.html + eks_managed_node_groups = { + example = { + desired_size = 1 + + instance_types = ["t3.large"] + labels = { + Example = "managed_node_groups" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + tags = { + ExtraTag = "example" + } + } + } + + fargate_profiles = { + default = { + name = "default" + selectors = [ + { + namespace = "backend" + labels = { + Application = "backend" + } + }, + { + namespace = "default" + labels = { + WorkerType = "fargate" + } + } + ] + + tags = { + Owner = "default" + } + + timeouts = { + create = "20m" + delete = "20m" + } + } + + secondary = { + name = "secondary" + selectors = [ + { + namespace = "default" + labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + } + ] + + # Using specific subnets instead of the subnets supplied for the cluster itself + subnet_ids = [module.vpc.private_subnets[1]] + + tags = { + Owner = "secondary" + } + } + } + + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + + azs = ["${local.region}a", "${local.region}b", "${local.region}c"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + enable_flow_log = true + create_flow_log_cloudwatch_iam_role = true + create_flow_log_cloudwatch_log_group = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} + +resource "aws_kms_key" "eks" { + description = "EKS Secret Encryption Key" + deletion_window_in_days = 7 + enable_key_rotation = true + + tags = local.tags +} diff --git a/examples/fargate_profile/outputs.tf b/examples/fargate_profile/outputs.tf new file mode 100644 index 0000000000..3e9620157b --- /dev/null +++ b/examples/fargate_profile/outputs.tf @@ -0,0 +1,167 @@ +################################################################################ +# Cluster +################################################################################ + +output "cluster_arn" { + description = "The Amazon Resource Name (ARN) of the cluster" + value = module.eks.cluster_arn +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded certificate data required to communicate with the cluster" + value = module.eks.cluster_certificate_authority_data +} + +output "cluster_endpoint" { + description = "Endpoint for your Kubernetes API server" + value = module.eks.cluster_endpoint +} + +output "cluster_id" { + description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_id +} + +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_oidc_issuer_url +} + +output "cluster_platform_version" { + description = "Platform version for the cluster" + value = module.eks.cluster_platform_version +} + +output "cluster_status" { + description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" + value = module.eks.cluster_status +} + +output "cluster_primary_security_group_id" { + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = module.eks.cluster_primary_security_group_id +} + +################################################################################ +# Security Group +################################################################################ + +output "cluster_security_group_arn" { + description = "Amazon Resource Name (ARN) of the cluster security group" + value = module.eks.cluster_security_group_arn +} + +output "cluster_security_group_id" { + description = "ID of the cluster security group" + value = module.eks.cluster_security_group_id +} + +################################################################################ +# Node Security Group +################################################################################ + +output "node_security_group_arn" { + description = "Amazon Resource Name (ARN) of the node shared security group" + value = module.eks.node_security_group_arn +} + +output "node_security_group_id" { + description = "ID of the node shared security group" + value = module.eks.node_security_group_id +} + +################################################################################ +# IRSA +################################################################################ + +output "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + value = module.eks.oidc_provider_arn +} + +################################################################################ +# IAM Role +################################################################################ + +output "cluster_iam_role_name" { + description = "IAM role name of the EKS cluster" + value = module.eks.cluster_iam_role_name +} + +output "cluster_iam_role_arn" { + description = "IAM role ARN of the EKS cluster" + value = module.eks.cluster_iam_role_arn +} + +output "cluster_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.cluster_iam_role_unique_id +} + +################################################################################ +# EKS Addons +################################################################################ + +output "cluster_addons" { + description = "Map of attribute maps for all EKS cluster addons enabled" + value = module.eks.cluster_addons +} + +################################################################################ +# EKS Identity Provider +################################################################################ + +output "cluster_identity_providers" { + description = "Map of attribute maps for all EKS identity providers enabled" + value = module.eks.cluster_identity_providers +} + +################################################################################ +# CloudWatch Log Group +################################################################################ + +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_name +} + +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_arn +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.eks.fargate_profiles +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +output "eks_managed_node_groups" { + description = "Map of attribute maps for all EKS managed node groups created" + value = module.eks.eks_managed_node_groups +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +output "self_managed_node_groups" { + description = "Map of attribute maps for all self managed node groups created" + value = module.eks.self_managed_node_groups +} + +################################################################################ +# Additional +################################################################################ + +output "aws_auth_configmap_yaml" { + description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" + value = module.eks.aws_auth_configmap_yaml +} diff --git a/examples/fargate/variables.tf b/examples/fargate_profile/variables.tf similarity index 100% rename from examples/fargate/variables.tf rename to examples/fargate_profile/variables.tf diff --git a/modules/fargate/versions.tf b/examples/fargate_profile/versions.tf similarity index 82% rename from modules/fargate/versions.tf rename to examples/fargate_profile/versions.tf index 9480a77da8..bfce6ae345 100644 --- a/modules/fargate/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.56" + version = ">= 3.64" } } } diff --git a/examples/instance_refresh/README.md b/examples/instance_refresh/README.md deleted file mode 100644 index cc558693ae..0000000000 --- a/examples/instance_refresh/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Instance refresh example - -This is EKS example using [instance refresh](https://aws.amazon.com/blogs/compute/introducing-instance-refresh-for-ec2-auto-scaling/) feature for worker groups. - -See [the official documentation](https://docs.aws.amazon.com/autoscaling/ec2/userguide/asg-instance-refresh.html) for more details. - -## Usage - -To run this example you need to execute: - -```bash -$ terraform init -$ terraform plan -$ terraform apply -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [helm](#requirement\_helm) | >= 2.0 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | -| [local](#requirement\_local) | >= 1.4 | -| [random](#requirement\_random) | >= 2.1 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [helm](#provider\_helm) | >= 2.0 | -| [random](#provider\_random) | >= 2.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [aws\_node\_termination\_handler\_role](#module\_aws\_node\_termination\_handler\_role) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | 4.1.0 | -| [aws\_node\_termination\_handler\_sqs](#module\_aws\_node\_termination\_handler\_sqs) | terraform-aws-modules/sqs/aws | ~> 3.0.0 | -| [eks](#module\_eks) | ../.. | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_autoscaling_lifecycle_hook.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_lifecycle_hook) | resource | -| [aws_cloudwatch_event_rule.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | -| [aws_cloudwatch_event_rule.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | -| [aws_cloudwatch_event_target.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | -| [aws_cloudwatch_event_target.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | -| [aws_iam_policy.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [helm_release.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | -| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | -| [aws_iam_policy_document.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.aws_node_termination_handler_events](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | - -## Inputs - -No inputs. - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | -| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | -| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | -| [sqs\_queue\_asg\_notification\_arn](#output\_sqs\_queue\_asg\_notification\_arn) | SQS queue ASG notification ARN | -| [sqs\_queue\_asg\_notification\_url](#output\_sqs\_queue\_asg\_notification\_url) | SQS queue ASG notification URL | - diff --git a/examples/instance_refresh/main.tf b/examples/instance_refresh/main.tf deleted file mode 100644 index f32964b1d7..0000000000 --- a/examples/instance_refresh/main.tf +++ /dev/null @@ -1,306 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - name = "instance_refresh-${random_string.suffix.result}" - cluster_version = "1.20" - region = "eu-west-1" -} - -################################################################################ -# EKS Module -################################################################################ - -# Based on the official aws-node-termination-handler setup guide at https://github.com/aws/aws-node-termination-handler#infrastructure-setup - -provider "helm" { - kubernetes { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token - } -} - -data "aws_caller_identity" "current" {} - -data "aws_iam_policy_document" "aws_node_termination_handler" { - statement { - effect = "Allow" - actions = [ - "ec2:DescribeInstances", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeTags", - ] - resources = [ - "*", - ] - } - statement { - effect = "Allow" - actions = [ - "autoscaling:CompleteLifecycleAction", - ] - resources = module.eks.workers_asg_arns - } - statement { - effect = "Allow" - actions = [ - "sqs:DeleteMessage", - "sqs:ReceiveMessage" - ] - resources = [ - module.aws_node_termination_handler_sqs.sqs_queue_arn - ] - } -} - -resource "aws_iam_policy" "aws_node_termination_handler" { - name = "${local.name}-aws-node-termination-handler" - policy = data.aws_iam_policy_document.aws_node_termination_handler.json -} - -data "aws_region" "current" {} - -data "aws_iam_policy_document" "aws_node_termination_handler_events" { - statement { - effect = "Allow" - principals { - type = "Service" - identifiers = [ - "events.amazonaws.com", - "sqs.amazonaws.com", - ] - } - actions = [ - "sqs:SendMessage", - ] - resources = [ - "arn:aws:sqs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${local.name}", - ] - } -} - -module "aws_node_termination_handler_sqs" { - source = "terraform-aws-modules/sqs/aws" - version = "~> 3.0.0" - name = local.name - message_retention_seconds = 300 - policy = data.aws_iam_policy_document.aws_node_termination_handler_events.json -} - -resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_asg" { - name = "${local.name}-asg-termination" - description = "Node termination event rule" - event_pattern = jsonencode( - { - "source" : [ - "aws.autoscaling" - ], - "detail-type" : [ - "EC2 Instance-terminate Lifecycle Action" - ] - "resources" : module.eks.workers_asg_arns - } - ) -} - -resource "aws_cloudwatch_event_target" "aws_node_termination_handler_asg" { - target_id = "${local.name}-asg-termination" - rule = aws_cloudwatch_event_rule.aws_node_termination_handler_asg.name - arn = module.aws_node_termination_handler_sqs.sqs_queue_arn -} - -resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_spot" { - name = "${local.name}-spot-termination" - description = "Node termination event rule" - event_pattern = jsonencode( - { - "source" : [ - "aws.ec2" - ], - "detail-type" : [ - "EC2 Spot Instance Interruption Warning" - ] - "resources" : module.eks.workers_asg_arns - } - ) -} - -resource "aws_cloudwatch_event_target" "aws_node_termination_handler_spot" { - target_id = "${local.name}-spot-termination" - rule = aws_cloudwatch_event_rule.aws_node_termination_handler_spot.name - arn = module.aws_node_termination_handler_sqs.sqs_queue_arn -} - -module "aws_node_termination_handler_role" { - source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "4.1.0" - create_role = true - role_description = "IRSA role for ANTH, cluster ${local.name}" - role_name_prefix = local.name - provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") - role_policy_arns = [aws_iam_policy.aws_node_termination_handler.arn] - oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:aws-node-termination-handler"] -} - -resource "helm_release" "aws_node_termination_handler" { - depends_on = [ - module.eks - ] - - name = "aws-node-termination-handler" - namespace = "kube-system" - repository = "https://aws.github.io/eks-charts" - chart = "aws-node-termination-handler" - version = "0.15.0" - create_namespace = true - - set { - name = "awsRegion" - value = data.aws_region.current.name - } - set { - name = "serviceAccount.name" - value = "aws-node-termination-handler" - } - set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.aws_node_termination_handler_role.iam_role_arn - type = "string" - } - set { - name = "enableSqsTerminationDraining" - value = "true" - } - set { - name = "enableSpotInterruptionDraining" - value = "true" - } - set { - name = "queueURL" - value = module.aws_node_termination_handler_sqs.sqs_queue_id - } - set { - name = "logLevel" - value = "debug" - } -} - -# Creating the lifecycle-hook outside of the ASG resource's `initial_lifecycle_hook` -# ensures that node termination does not require the lifecycle action to be completed, -# and thus allows the ASG to be destroyed cleanly. -resource "aws_autoscaling_lifecycle_hook" "aws_node_termination_handler" { - count = length(module.eks.workers_asg_names) - name = "aws-node-termination-handler" - autoscaling_group_name = module.eks.workers_asg_names[count.index] - lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" - heartbeat_timeout = 300 - default_result = "CONTINUE" -} - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_version = local.cluster_version - - vpc_id = module.vpc.vpc_id - subnets = module.vpc.private_subnets - - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - - enable_irsa = true - worker_groups_launch_template = [ - { - name = "refresh" - asg_max_size = 2 - asg_desired_capacity = 2 - instance_refresh_enabled = true - instance_refresh_instance_warmup = 60 - public_ip = true - metadata_http_put_response_hop_limit = 3 - update_default_version = true - instance_refresh_triggers = ["tag"] - tags = [ - { - key = "aws-node-termination-handler/managed" - value = "" - propagate_at_launch = true - }, - { - key = "foo" - value = "buzz" - propagate_at_launch = true - } - ] - } - ] - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# Kubernetes provider configuration -################################################################################ - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} - -################################################################################ -# Supporting Resources -################################################################################ - -data "aws_availability_zones" "available" { -} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} diff --git a/examples/instance_refresh/outputs.tf b/examples/instance_refresh/outputs.tf deleted file mode 100644 index a3dd033616..0000000000 --- a/examples/instance_refresh/outputs.tf +++ /dev/null @@ -1,29 +0,0 @@ -output "cluster_endpoint" { - description = "Endpoint for EKS control plane." - value = module.eks.cluster_endpoint -} - -output "cluster_security_group_id" { - description = "Security group ids attached to the cluster control plane." - value = module.eks.cluster_security_group_id -} - -output "kubectl_config" { - description = "kubectl config as generated by the module." - value = module.eks.kubeconfig -} - -output "config_map_aws_auth" { - description = "A kubernetes configuration to authenticate to this EKS cluster." - value = module.eks.config_map_aws_auth -} - -output "sqs_queue_asg_notification_arn" { - description = "SQS queue ASG notification ARN" - value = module.aws_node_termination_handler_sqs.sqs_queue_arn -} - -output "sqs_queue_asg_notification_url" { - description = "SQS queue ASG notification URL" - value = module.aws_node_termination_handler_sqs.sqs_queue_id -} diff --git a/examples/instance_refresh/versions.tf b/examples/instance_refresh/versions.tf deleted file mode 100644 index 67c2d66a82..0000000000 --- a/examples/instance_refresh/versions.tf +++ /dev/null @@ -1,26 +0,0 @@ -terraform { - required_version = ">= 0.13.1" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.56" - } - local = { - source = "hashicorp/local" - version = ">= 1.4" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 1.11.1" - } - random = { - source = "hashicorp/random" - version = ">= 2.1" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.0" - } - } -} diff --git a/examples/irsa/README.md b/examples/irsa/README.md deleted file mode 100644 index 137f3d63d8..0000000000 --- a/examples/irsa/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# IAM Roles for Service Accounts - -This example shows how to create an IAM role to be used for a Kubernetes `ServiceAccount`. It will create a policy and role to be used by the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) using the [public Helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler). - -See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) for more details. - -## Usage - -To run this example you need to execute: - -```bash -$ terraform init -$ terraform plan -$ terraform apply -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [helm](#requirement\_helm) | >= 2.0 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | -| [local](#requirement\_local) | >= 1.4 | -| [random](#requirement\_random) | >= 2.1 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [helm](#provider\_helm) | >= 2.0 | -| [random](#provider\_random) | >= 2.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | ../.. | n/a | -| [iam\_assumable\_role\_admin](#module\_iam\_assumable\_role\_admin) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | ~> 4.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_iam_policy.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [helm_release.cluster-autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | -| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | -| [aws_iam_policy_document.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | - -## Inputs - -No inputs. - -## Outputs - -| Name | Description | -|------|-------------| -| [aws\_account\_id](#output\_aws\_account\_id) | IAM AWS account id | - diff --git a/examples/irsa/irsa.tf b/examples/irsa/irsa.tf deleted file mode 100644 index a36d0e3394..0000000000 --- a/examples/irsa/irsa.tf +++ /dev/null @@ -1,114 +0,0 @@ -data "aws_caller_identity" "current" {} - -data "aws_region" "current" {} - -locals { - k8s_service_account_namespace = "kube-system" - k8s_service_account_name = "cluster-autoscaler-aws" -} - -provider "helm" { - kubernetes { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token - } -} - -resource "helm_release" "cluster-autoscaler" { - depends_on = [ - module.eks - ] - - name = "cluster-autoscaler" - namespace = local.k8s_service_account_namespace - repository = "https://kubernetes.github.io/autoscaler" - chart = "cluster-autoscaler" - version = "9.10.7" - create_namespace = false - - set { - name = "awsRegion" - value = data.aws_region.current.name - } - set { - name = "rbac.serviceAccount.name" - value = local.k8s_service_account_name - } - set { - name = "rbac.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.iam_assumable_role_admin.iam_role_arn - type = "string" - } - set { - name = "autoDiscovery.clusterName" - value = local.name - } - set { - name = "autoDiscovery.enabled" - value = "true" - } - set { - name = "rbac.create" - value = "true" - } -} - -module "iam_assumable_role_admin" { - source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "~> 4.0" - - create_role = true - role_name = "cluster-autoscaler" - provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") - role_policy_arns = [aws_iam_policy.cluster_autoscaler.arn] - oidc_fully_qualified_subjects = ["system:serviceaccount:${local.k8s_service_account_namespace}:${local.k8s_service_account_name}"] -} - -resource "aws_iam_policy" "cluster_autoscaler" { - name_prefix = "cluster-autoscaler" - description = "EKS cluster-autoscaler policy for cluster ${module.eks.cluster_id}" - policy = data.aws_iam_policy_document.cluster_autoscaler.json -} - -data "aws_iam_policy_document" "cluster_autoscaler" { - statement { - sid = "clusterAutoscalerAll" - effect = "Allow" - - actions = [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "ec2:DescribeLaunchTemplateVersions", - ] - - resources = ["*"] - } - - statement { - sid = "clusterAutoscalerOwn" - effect = "Allow" - - actions = [ - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - ] - - resources = ["*"] - - condition { - test = "StringEquals" - variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/${module.eks.cluster_id}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled" - values = ["true"] - } - } -} diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf deleted file mode 100644 index ebe2eeffea..0000000000 --- a/examples/irsa/main.tf +++ /dev/null @@ -1,115 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - name = "irsa-${random_string.suffix.result}" - cluster_version = "1.20" - region = "eu-west-1" -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_version = local.cluster_version - - vpc_id = module.vpc.vpc_id - subnets = module.vpc.private_subnets - - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - - enable_irsa = true - - worker_groups = [ - { - name = "worker-group-1" - instance_type = "t3.medium" - asg_desired_capacity = 1 - asg_max_size = 4 - tags = [ - { - "key" = "k8s.io/cluster-autoscaler/enabled" - "propagate_at_launch" = "false" - "value" = "true" - }, - { - "key" = "k8s.io/cluster-autoscaler/${local.name}" - "propagate_at_launch" = "false" - "value" = "owned" - } - ] - } - ] - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# Kubernetes provider configuration -################################################################################ - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} - -################################################################################ -# Supporting Resources -################################################################################ - -data "aws_availability_zones" "available" { -} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - diff --git a/examples/irsa/outputs.tf b/examples/irsa/outputs.tf deleted file mode 100644 index 796e8ee3d4..0000000000 --- a/examples/irsa/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "aws_account_id" { - description = "IAM AWS account id" - value = data.aws_caller_identity.current.account_id -} diff --git a/examples/irsa/versions.tf b/examples/irsa/versions.tf deleted file mode 100644 index 67c2d66a82..0000000000 --- a/examples/irsa/versions.tf +++ /dev/null @@ -1,26 +0,0 @@ -terraform { - required_version = ">= 0.13.1" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.56" - } - local = { - source = "hashicorp/local" - version = ">= 1.4" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 1.11.1" - } - random = { - source = "hashicorp/random" - version = ">= 2.1" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.0" - } - } -} diff --git a/examples/irsa_autoscale_refresh/README.md b/examples/irsa_autoscale_refresh/README.md new file mode 100644 index 0000000000..a21f31f6d3 --- /dev/null +++ b/examples/irsa_autoscale_refresh/README.md @@ -0,0 +1,102 @@ +# IRSA, Cluster Autoscaler, and Instance Refresh example + +Configuration in this directory creates an AWS EKS cluster with: +- [IAM Roles for Service Accounts (IRSA)](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) enabled +- [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md) provisioned via a Helm Chart manifest +- [Instance Refresh](https://aws.amazon.com/blogs/compute/introducing-instance-refresh-for-ec2-auto-scaling/) feature for self managed node groups +- [Node Termination Handler](https://github.com/aws/aws-node-termination-handler) provisioned via a Helm Chart manifest + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.64 | +| [helm](#requirement\_helm) | >= 2.0 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.64 | +| [helm](#provider\_helm) | >= 2.0 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [aws\_node\_termination\_handler\_role](#module\_aws\_node\_termination\_handler\_role) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | ~> 4.0 | +| [aws\_node\_termination\_handler\_sqs](#module\_aws\_node\_termination\_handler\_sqs) | terraform-aws-modules/sqs/aws | ~> 3.0 | +| [eks](#module\_eks) | ../.. | n/a | +| [iam\_assumable\_role\_cluster\_autoscaler](#module\_iam\_assumable\_role\_cluster\_autoscaler) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | ~> 4.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_autoscaling_lifecycle_hook.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_lifecycle_hook) | resource | +| [aws_cloudwatch_event_rule.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_rule.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_target.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_cloudwatch_event_target.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_iam_policy.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [helm_release.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [null_resource.apply](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [aws_iam_policy_document.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.aws_node_termination_handler_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | +| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | +| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | +| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | +| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | +| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | +| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | +| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | +| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | + diff --git a/examples/irsa_autoscale_refresh/charts.tf b/examples/irsa_autoscale_refresh/charts.tf new file mode 100644 index 0000000000..1ef1195b9e --- /dev/null +++ b/examples/irsa_autoscale_refresh/charts.tf @@ -0,0 +1,294 @@ +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.cluster.token + } +} + +################################################################################ +# Cluster Autoscaler +# Based on the official docs at +# https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler +################################################################################ + +resource "helm_release" "cluster_autoscaler" { + name = "cluster-autoscaler" + namespace = "kube-system" + repository = "https://kubernetes.github.io/autoscaler" + chart = "cluster-autoscaler" + version = "9.10.8" + create_namespace = false + + set { + name = "awsRegion" + value = local.region + } + + set { + name = "rbac.serviceAccount.name" + value = "cluster-autoscaler-aws" + } + + set { + name = "rbac.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.iam_assumable_role_cluster_autoscaler.iam_role_arn + type = "string" + } + + set { + name = "autoDiscovery.clusterName" + value = local.name + } + + set { + name = "autoDiscovery.enabled" + value = "true" + } + + set { + name = "rbac.create" + value = "true" + } + + depends_on = [ + module.eks.cluster_id, + null_resource.apply, + ] +} + +module "iam_assumable_role_cluster_autoscaler" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "~> 4.0" + + create_role = true + role_name_prefix = "cluster-autoscaler" + role_description = "IRSA role for cluster autoscaler" + + provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") + role_policy_arns = [aws_iam_policy.cluster_autoscaler.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:cluster-autoscaler-aws"] + oidc_fully_qualified_audiences = ["sts.amazonaws.com"] + + tags = local.tags +} + +resource "aws_iam_policy" "cluster_autoscaler" { + name = "KarpenterControllerPolicy-refresh" + policy = data.aws_iam_policy_document.cluster_autoscaler.json + + tags = local.tags +} + +data "aws_iam_policy_document" "cluster_autoscaler" { + statement { + sid = "clusterAutoscalerAll" + actions = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions", + ] + resources = ["*"] + } + + statement { + sid = "clusterAutoscalerOwn" + actions = [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup", + ] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/${module.eks.cluster_id}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled" + values = ["true"] + } + } +} + +################################################################################ +# Node Termination Handler +# Based on the official docs at +# https://github.com/aws/aws-node-termination-handler +################################################################################ + +resource "helm_release" "aws_node_termination_handler" { + name = "aws-node-termination-handler" + namespace = "kube-system" + repository = "https://aws.github.io/eks-charts" + chart = "aws-node-termination-handler" + version = "0.16.0" + create_namespace = false + + set { + name = "awsRegion" + value = local.region + } + + set { + name = "serviceAccount.name" + value = "aws-node-termination-handler" + } + + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.aws_node_termination_handler_role.iam_role_arn + type = "string" + } + + set { + name = "enableSqsTerminationDraining" + value = "true" + } + + set { + name = "enableSpotInterruptionDraining" + value = "true" + } + + set { + name = "queueURL" + value = module.aws_node_termination_handler_sqs.sqs_queue_id + } + + set { + name = "logLevel" + value = "debug" + } + + depends_on = [ + module.eks.cluster_id, + null_resource.apply, + ] +} + +module "aws_node_termination_handler_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "~> 4.0" + + create_role = true + role_name_prefix = "node-termination-handler" + role_description = "IRSA role for node termination handler" + + provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") + role_policy_arns = [aws_iam_policy.aws_node_termination_handler.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:aws-node-termination-handler"] + oidc_fully_qualified_audiences = ["sts.amazonaws.com"] + + tags = local.tags +} + +resource "aws_iam_policy" "aws_node_termination_handler" { + name = "${local.name}-aws-node-termination-handler" + policy = data.aws_iam_policy_document.aws_node_termination_handler.json + + tags = local.tags +} + +data "aws_iam_policy_document" "aws_node_termination_handler" { + statement { + actions = [ + "ec2:DescribeInstances", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeTags", + ] + resources = ["*"] + } + + statement { + actions = ["autoscaling:CompleteLifecycleAction"] + resources = [for group in module.eks.self_managed_node_groups : group.autoscaling_group_arn] + } + + statement { + actions = [ + "sqs:DeleteMessage", + "sqs:ReceiveMessage" + ] + resources = [module.aws_node_termination_handler_sqs.sqs_queue_arn] + } +} + +module "aws_node_termination_handler_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "~> 3.0" + + name = local.name + message_retention_seconds = 300 + policy = data.aws_iam_policy_document.aws_node_termination_handler_sqs.json + + tags = local.tags +} + +data "aws_iam_policy_document" "aws_node_termination_handler_sqs" { + statement { + actions = ["sqs:SendMessage"] + resources = ["arn:aws:sqs:${local.region}:${data.aws_caller_identity.current.account_id}:${local.name}"] + + principals { + type = "Service" + identifiers = [ + "events.amazonaws.com", + "sqs.amazonaws.com", + ] + } + } +} + +resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_asg" { + name = "${local.name}-asg-termination" + description = "Node termination event rule" + + event_pattern = jsonencode({ + "source" : ["aws.autoscaling"], + "detail-type" : ["EC2 Instance-terminate Lifecycle Action"] + "resources" : [for group in module.eks.self_managed_node_groups : group.autoscaling_group_arn] + }) + + tags = local.tags +} + +resource "aws_cloudwatch_event_target" "aws_node_termination_handler_asg" { + target_id = "${local.name}-asg-termination" + rule = aws_cloudwatch_event_rule.aws_node_termination_handler_asg.name + arn = module.aws_node_termination_handler_sqs.sqs_queue_arn +} + +resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_spot" { + name = "${local.name}-spot-termination" + description = "Node termination event rule" + event_pattern = jsonencode({ + "source" : ["aws.ec2"], + "detail-type" : ["EC2 Spot Instance Interruption Warning"] + "resources" : [for group in module.eks.self_managed_node_groups : group.autoscaling_group_arn] + }) +} + +resource "aws_cloudwatch_event_target" "aws_node_termination_handler_spot" { + target_id = "${local.name}-spot-termination" + rule = aws_cloudwatch_event_rule.aws_node_termination_handler_spot.name + arn = module.aws_node_termination_handler_sqs.sqs_queue_arn +} + +# Creating the lifecycle-hook outside of the ASG resource's `initial_lifecycle_hook` +# ensures that node termination does not require the lifecycle action to be completed, +# and thus allows the ASG to be destroyed cleanly. +resource "aws_autoscaling_lifecycle_hook" "aws_node_termination_handler" { + for_each = module.eks.self_managed_node_groups + + name = "aws-node-termination-handler-${each.value.autoscaling_group_name}" + autoscaling_group_name = each.value.autoscaling_group_name + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + heartbeat_timeout = 300 + default_result = "CONTINUE" +} diff --git a/examples/irsa_autoscale_refresh/main.tf b/examples/irsa_autoscale_refresh/main.tf new file mode 100644 index 0000000000..9e74e3d9fe --- /dev/null +++ b/examples/irsa_autoscale_refresh/main.tf @@ -0,0 +1,203 @@ +provider "aws" { + region = local.region +} + +locals { + name = "ex-${replace(basename(path.cwd), "_", "-")}" + cluster_version = "1.21" + region = "eu-west-1" + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +data "aws_caller_identity" "current" {} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + enable_irsa = true + + # Self Managed Node Group(s) + self_managed_node_groups = { + refresh = { + max_size = 5 + desired_size = 1 + + instance_type = "m5.large" + + instance_refresh = { + strategy = "Rolling" + preferences = { + checkpoint_delay = 600 + checkpoint_percentages = [35, 70, 100] + instance_warmup = 300 + min_healthy_percentage = 50 + } + triggers = ["tag"] + } + + propogate_tags = [{ + key = "aws-node-termination-handler/managed" + value = true + propagate_at_launch = true + }] + } + + mixed_instance = { + use_mixed_instances_policy = true + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 10 + spot_allocation_strategy = "capacity-optimized" + } + + override = [ + { + instance_type = "m5.large" + weighted_capacity = "1" + }, + { + instance_type = "m6i.large" + weighted_capacity = "2" + }, + ] + } + + propogate_tags = [{ + key = "aws-node-termination-handler/managed" + value = true + propagate_at_launch = true + }] + } + + spot = { + instance_type = "m5.large" + instance_market_options = { + market_type = "spot" + } + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + propogate_tags = [{ + key = "aws-node-termination-handler/managed" + value = true + propagate_at_launch = true + }] + } + } + + tags = merge(local.tags, { Foo = "bar" }) +} + +################################################################################ +# aws-auth configmap +# Only EKS managed node groups automatically add roles to aws-auth configmap +# so we need to ensure fargate profiles and self-managed node roles are added +################################################################################ + +data "aws_eks_cluster_auth" "this" { + name = module.eks.cluster_id +} + +locals { + kubeconfig = yamlencode({ + apiVersion = "v1" + kind = "Config" + current-context = "terraform" + clusters = [{ + name = module.eks.cluster_id + cluster = { + certificate-authority-data = module.eks.cluster_certificate_authority_data + server = module.eks.cluster_endpoint + } + }] + contexts = [{ + name = "terraform" + context = { + cluster = module.eks.cluster_id + user = "terraform" + } + }] + users = [{ + name = "terraform" + user = { + token = data.aws_eks_cluster_auth.this.token + } + }] + }) +} + +resource "null_resource" "apply" { + triggers = { + kubeconfig = base64encode(local.kubeconfig) + cmd_patch = <<-EOT + kubectl create configmap aws-auth -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) + kubectl patch configmap/aws-auth --patch "${module.eks.aws_auth_configmap_yaml}" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) + EOT + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = self.triggers.kubeconfig + } + command = self.triggers.cmd_patch + } +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + + azs = ["${local.region}a", "${local.region}b", "${local.region}c"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + enable_flow_log = true + create_flow_log_cloudwatch_iam_role = true + create_flow_log_cloudwatch_log_group = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/examples/irsa_autoscale_refresh/outputs.tf b/examples/irsa_autoscale_refresh/outputs.tf new file mode 100644 index 0000000000..3e9620157b --- /dev/null +++ b/examples/irsa_autoscale_refresh/outputs.tf @@ -0,0 +1,167 @@ +################################################################################ +# Cluster +################################################################################ + +output "cluster_arn" { + description = "The Amazon Resource Name (ARN) of the cluster" + value = module.eks.cluster_arn +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded certificate data required to communicate with the cluster" + value = module.eks.cluster_certificate_authority_data +} + +output "cluster_endpoint" { + description = "Endpoint for your Kubernetes API server" + value = module.eks.cluster_endpoint +} + +output "cluster_id" { + description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_id +} + +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_oidc_issuer_url +} + +output "cluster_platform_version" { + description = "Platform version for the cluster" + value = module.eks.cluster_platform_version +} + +output "cluster_status" { + description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" + value = module.eks.cluster_status +} + +output "cluster_primary_security_group_id" { + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = module.eks.cluster_primary_security_group_id +} + +################################################################################ +# Security Group +################################################################################ + +output "cluster_security_group_arn" { + description = "Amazon Resource Name (ARN) of the cluster security group" + value = module.eks.cluster_security_group_arn +} + +output "cluster_security_group_id" { + description = "ID of the cluster security group" + value = module.eks.cluster_security_group_id +} + +################################################################################ +# Node Security Group +################################################################################ + +output "node_security_group_arn" { + description = "Amazon Resource Name (ARN) of the node shared security group" + value = module.eks.node_security_group_arn +} + +output "node_security_group_id" { + description = "ID of the node shared security group" + value = module.eks.node_security_group_id +} + +################################################################################ +# IRSA +################################################################################ + +output "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + value = module.eks.oidc_provider_arn +} + +################################################################################ +# IAM Role +################################################################################ + +output "cluster_iam_role_name" { + description = "IAM role name of the EKS cluster" + value = module.eks.cluster_iam_role_name +} + +output "cluster_iam_role_arn" { + description = "IAM role ARN of the EKS cluster" + value = module.eks.cluster_iam_role_arn +} + +output "cluster_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.cluster_iam_role_unique_id +} + +################################################################################ +# EKS Addons +################################################################################ + +output "cluster_addons" { + description = "Map of attribute maps for all EKS cluster addons enabled" + value = module.eks.cluster_addons +} + +################################################################################ +# EKS Identity Provider +################################################################################ + +output "cluster_identity_providers" { + description = "Map of attribute maps for all EKS identity providers enabled" + value = module.eks.cluster_identity_providers +} + +################################################################################ +# CloudWatch Log Group +################################################################################ + +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_name +} + +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_arn +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.eks.fargate_profiles +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +output "eks_managed_node_groups" { + description = "Map of attribute maps for all EKS managed node groups created" + value = module.eks.eks_managed_node_groups +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +output "self_managed_node_groups" { + description = "Map of attribute maps for all self managed node groups created" + value = module.eks.self_managed_node_groups +} + +################################################################################ +# Additional +################################################################################ + +output "aws_auth_configmap_yaml" { + description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" + value = module.eks.aws_auth_configmap_yaml +} diff --git a/examples/instance_refresh/variables.tf b/examples/irsa_autoscale_refresh/variables.tf similarity index 100% rename from examples/instance_refresh/variables.tf rename to examples/irsa_autoscale_refresh/variables.tf diff --git a/examples/irsa_autoscale_refresh/versions.tf b/examples/irsa_autoscale_refresh/versions.tf new file mode 100644 index 0000000000..4706dec92a --- /dev/null +++ b/examples/irsa_autoscale_refresh/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.64" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.0" + } + } +} diff --git a/examples/launch_templates/README.md b/examples/launch_templates/README.md deleted file mode 100644 index dbd5073f89..0000000000 --- a/examples/launch_templates/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Launch templates example - -This is EKS example using workers launch template with worker groups feature. - -See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) for more details. - -## Usage - -To run this example you need to execute: - -```bash -$ terraform init -$ terraform plan -$ terraform apply -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | -| [local](#requirement\_local) | >= 1.4 | -| [random](#requirement\_random) | >= 2.1 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [random](#provider\_random) | >= 2.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | ../.. | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | - -## Resources - -| Name | Type | -|------|------| -| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | -| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | - -## Inputs - -No inputs. - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | -| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | -| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | - diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf deleted file mode 100644 index 476ca13d68..0000000000 --- a/examples/launch_templates/main.tf +++ /dev/null @@ -1,134 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - name = "launch_template-${random_string.suffix.result}" - cluster_version = "1.20" - region = "eu-west-1" -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - vpc_id = module.vpc.vpc_id - subnets = module.vpc.private_subnets - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - - worker_groups_launch_template = [ - { - name = "worker-group-1" - instance_type = "t3.small" - asg_desired_capacity = 2 - public_ip = true - tags = [{ - key = "ExtraTag" - value = "TagValue" - propagate_at_launch = true - }] - }, - { - name = "worker-group-2" - instance_type = "t3.medium" - asg_desired_capacity = 1 - public_ip = true - ebs_optimized = true - }, - { - name = "worker-group-3" - instance_type = "t2.large" - asg_desired_capacity = 1 - public_ip = true - elastic_inference_accelerator = "eia2.medium" - }, - { - name = "worker-group-4" - instance_type = "t3.small" - asg_desired_capacity = 1 - public_ip = true - root_volume_size = 150 - root_volume_type = "gp3" - root_volume_throughput = 300 - additional_ebs_volumes = [ - { - block_device_name = "/dev/xvdb" - volume_size = 100 - volume_type = "gp3" - throughput = 150 - }, - ] - }, - ] - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# Kubernetes provider configuration -################################################################################ - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} - -################################################################################ -# Supporting Resources -################################################################################ - -data "aws_availability_zones" "available" { -} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} diff --git a/examples/launch_templates/outputs.tf b/examples/launch_templates/outputs.tf deleted file mode 100644 index b778ec7926..0000000000 --- a/examples/launch_templates/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -output "cluster_endpoint" { - description = "Endpoint for EKS control plane." - value = module.eks.cluster_endpoint -} - -output "cluster_security_group_id" { - description = "Security group ids attached to the cluster control plane." - value = module.eks.cluster_security_group_id -} - -output "kubectl_config" { - description = "kubectl config as generated by the module." - value = module.eks.kubeconfig -} - -output "config_map_aws_auth" { - description = "A kubernetes configuration to authenticate to this EKS cluster." - value = module.eks.config_map_aws_auth -} diff --git a/examples/launch_templates/pre_userdata.sh b/examples/launch_templates/pre_userdata.sh deleted file mode 100644 index 4cbf0d114b..0000000000 --- a/examples/launch_templates/pre_userdata.sh +++ /dev/null @@ -1 +0,0 @@ -yum update -y diff --git a/examples/launch_templates/versions.tf b/examples/launch_templates/versions.tf deleted file mode 100644 index 8e2b837984..0000000000 --- a/examples/launch_templates/versions.tf +++ /dev/null @@ -1,22 +0,0 @@ -terraform { - required_version = ">= 0.13.1" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.56" - } - local = { - source = "hashicorp/local" - version = ">= 1.4" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 1.11.1" - } - random = { - source = "hashicorp/random" - version = ">= 2.1" - } - } -} diff --git a/examples/launch_templates_with_managed_node_groups/README.md b/examples/launch_templates_with_managed_node_groups/README.md deleted file mode 100644 index 3cae549325..0000000000 --- a/examples/launch_templates_with_managed_node_groups/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Launch template with managed groups example - -This is EKS example using workers custom launch template with managed groups feature in two different ways: - -- Using a defined existing launch template created outside module -- Using dlaunch template which will be created by module with user customization - -See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html) for more details. - -## Usage - -To run this example you need to execute: - -```bash -$ terraform init -$ terraform plan -$ terraform apply -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | -| [local](#requirement\_local) | >= 1.4 | -| [random](#requirement\_random) | >= 2.1 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [random](#provider\_random) | >= 2.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | ../.. | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_iam_service_linked_role.autoscaling](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_service_linked_role) | resource | -| [aws_launch_template.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | -| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | -| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | - -## Inputs - -No inputs. - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | -| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | -| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | - diff --git a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf deleted file mode 100644 index 3f834ad100..0000000000 --- a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf +++ /dev/null @@ -1,78 +0,0 @@ -# if you have used ASGs before, that role got auto-created already and you need to import to TF state -resource "aws_iam_service_linked_role" "autoscaling" { - aws_service_name = "autoscaling.amazonaws.com" - description = "Default Service-Linked Role enables access to AWS Services and Resources used or managed by Auto Scaling" - custom_suffix = "lt_with_managed_node_groups" # the full name is "AWSServiceRoleForAutoScaling_lt_with_managed_node_groups" < 64 characters -} - -#data "aws_caller_identity" "current" {} -# -## This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes -#data "aws_iam_policy_document" "ebs_decryption" { -# # Copy of default KMS policy that lets you manage it -# statement { -# sid = "Enable IAM User Permissions" -# effect = "Allow" -# -# principals { -# type = "AWS" -# identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] -# } -# -# actions = [ -# "kms:*" -# ] -# -# resources = ["*"] -# } -# -# # Required for EKS -# statement { -# sid = "Allow service-linked role use of the CMK" -# effect = "Allow" -# -# principals { -# type = "AWS" -# identifiers = [ -# "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes -# module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs -# ] -# } -# -# actions = [ -# "kms:Encrypt", -# "kms:Decrypt", -# "kms:ReEncrypt*", -# "kms:GenerateDataKey*", -# "kms:DescribeKey" -# ] -# -# resources = ["*"] -# } -# -# statement { -# sid = "Allow attachment of persistent resources" -# effect = "Allow" -# -# principals { -# type = "AWS" -# identifiers = [ -# "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes -# module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs -# ] -# } -# -# actions = [ -# "kms:CreateGrant" -# ] -# -# resources = ["*"] -# -# condition { -# test = "Bool" -# variable = "kms:GrantIsForAWSResource" -# values = ["true"] -# } -# -# } -#} diff --git a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf deleted file mode 100644 index 0f0e4ebf31..0000000000 --- a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf +++ /dev/null @@ -1,98 +0,0 @@ -#data "template_file" "launch_template_userdata" { -# template = file("${path.module}/templates/userdata.sh.tpl") -# -# vars = { -# cluster_name = local.name -# endpoint = module.eks.cluster_endpoint -# cluster_auth_base64 = module.eks.cluster_certificate_authority_data -# -# bootstrap_extra_args = "" -# kubelet_extra_args = "" -# } -#} - -# This is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) -# there are several more options one could set but you probably dont need to modify them -# you can take the default and add your custom AMI and/or custom tags -# -# Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, -# then the default user-data for bootstrapping a cluster is merged in the copy. - -resource "aws_launch_template" "default" { - name_prefix = "eks-example-" - description = "Default Launch-Template" - update_default_version = true - - block_device_mappings { - device_name = "/dev/xvda" - - ebs { - volume_size = 100 - volume_type = "gp2" - delete_on_termination = true - # encrypted = true - - # Enable this if you want to encrypt your node root volumes with a KMS/CMK. encryption of PVCs is handled via k8s StorageClass tho - # you also need to attach data.aws_iam_policy_document.ebs_decryption.json from the disk_encryption_policy.tf to the KMS/CMK key then !! - # kms_key_id = var.kms_key_arn - } - } - - monitoring { - enabled = true - } - - network_interfaces { - associate_public_ip_address = false - delete_on_termination = true - security_groups = [module.eks.worker_security_group_id] - } - - # if you want to use a custom AMI - # image_id = var.ami_id - - # If you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then - # you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 - # - # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) - - # user_data = base64encode( - # data.template_file.launch_template_userdata.rendered, - # ) - - # Supplying custom tags to EKS instances is another use-case for LaunchTemplates - tag_specifications { - resource_type = "instance" - - tags = { - CustomTag = "Instance custom tag" - } - } - - # Supplying custom tags to EKS instances root volumes is another use-case for LaunchTemplates. (doesnt add tags to dynamically provisioned volumes via PVC) - tag_specifications { - resource_type = "volume" - - tags = { - CustomTag = "Volume custom tag" - } - } - - # Supplying custom tags to EKS instances ENI's is another use-case for LaunchTemplates - tag_specifications { - resource_type = "network-interface" - - tags = { - CustomTag = "EKS example" - } - } - - # Tag the LT itself - tags = { - CustomTag = "Launch template custom tag" - } - - lifecycle { - create_before_destroy = true - } -} diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf deleted file mode 100644 index 4392c606fa..0000000000 --- a/examples/launch_templates_with_managed_node_groups/main.tf +++ /dev/null @@ -1,149 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - name = "lt_with_mng-${random_string.suffix.result}" - cluster_version = "1.20" - region = "eu-west-1" -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_version = local.cluster_version - - vpc_id = module.vpc.vpc_id - subnets = module.vpc.private_subnets - - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - - node_groups = { - # use arleady defined launch template - example1 = { - name_prefix = "example1" - desired_capacity = 1 - max_capacity = 15 - min_capacity = 1 - - launch_template_id = aws_launch_template.default.id - launch_template_version = aws_launch_template.default.default_version - - instance_types = ["t3.small"] - - additional_tags = { - ExtraTag = "example1" - } - } - # create launch template - example2 = { - create_launch_template = true - desired_capacity = 1 - max_capacity = 10 - min_capacity = 1 - - disk_size = 50 - disk_type = "gp3" - disk_throughput = 150 - disk_iops = 3000 - - instance_types = ["t3.large"] - capacity_type = "SPOT" - - bootstrap_env = { - CONTAINER_RUNTIME = "containerd" - USE_MAX_PODS = false - } - kubelet_extra_args = "--max-pods=110" - k8s_labels = { - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - additional_tags = { - ExtraTag = "example2" - } - taints = [ - { - key = "dedicated" - value = "gpuGroup" - effect = "NO_SCHEDULE" - } - ] - update_config = { - max_unavailable_percentage = 50 # or set `max_unavailable` - } - } - } - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# Kubernetes provider configuration -################################################################################ - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} - -################################################################################ -# Supporting Resources -################################################################################ - -data "aws_availability_zones" "available" { -} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} diff --git a/examples/launch_templates_with_managed_node_groups/outputs.tf b/examples/launch_templates_with_managed_node_groups/outputs.tf deleted file mode 100644 index 359db3a481..0000000000 --- a/examples/launch_templates_with_managed_node_groups/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -output "cluster_endpoint" { - description = "Endpoint for EKS control plane." - value = module.eks.cluster_endpoint -} - -output "cluster_security_group_id" { - description = "Security group ids attached to the cluster control plane." - value = module.eks.cluster_security_group_id -} - -output "kubectl_config" { - description = "kubectl config as generated by the module." - value = module.eks.kubeconfig -} - -output "config_map_aws_auth" { - description = "A kubernetes configuration to authenticate to this EKS cluster." - value = module.eks.config_map_aws_auth -} diff --git a/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl b/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl deleted file mode 100644 index 41eeb0ba03..0000000000 --- a/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl +++ /dev/null @@ -1,12 +0,0 @@ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="//" - ---// -Content-Type: text/x-shellscript; charset="us-ascii" -#!/bin/bash -set -e - -# Bootstrap and join the cluster -/etc/eks/bootstrap.sh --b64-cluster-ca '${cluster_auth_base64}' --apiserver-endpoint '${endpoint}' ${bootstrap_extra_args} --kubelet-extra-args "${kubelet_extra_args}" '${cluster_name}' - ---//-- diff --git a/examples/launch_templates_with_managed_node_groups/variables.tf b/examples/launch_templates_with_managed_node_groups/variables.tf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/launch_templates_with_managed_node_groups/versions.tf b/examples/launch_templates_with_managed_node_groups/versions.tf deleted file mode 100644 index 8e2b837984..0000000000 --- a/examples/launch_templates_with_managed_node_groups/versions.tf +++ /dev/null @@ -1,22 +0,0 @@ -terraform { - required_version = ">= 0.13.1" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.56" - } - local = { - source = "hashicorp/local" - version = ">= 1.4" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 1.11.1" - } - random = { - source = "hashicorp/random" - version = ">= 2.1" - } - } -} diff --git a/examples/managed_node_groups/README.md b/examples/managed_node_groups/README.md deleted file mode 100644 index 7121431392..0000000000 --- a/examples/managed_node_groups/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Managed groups example - -This is EKS example using managed groups feature in two different ways: - -- Using SPOT instances in node group -- Using ON_DEMAND instance in node group - -See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for more details. - -## Usage - -To run this example you need to execute: - -```bash -$ terraform init -$ terraform plan -$ terraform apply -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | -| [local](#requirement\_local) | >= 1.4 | -| [random](#requirement\_random) | >= 2.1 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [random](#provider\_random) | >= 2.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | ../.. | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | - -## Resources - -| Name | Type | -|------|------| -| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | -| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [map\_accounts](#input\_map\_accounts) | Additional AWS account numbers to add to the aws-auth configmap. | `list(string)` |
[
"777777777777",
"888888888888"
]
| no | -| [map\_roles](#input\_map\_roles) | Additional IAM roles to add to the aws-auth configmap. |
list(object({
rolearn = string
username = string
groups = list(string)
}))
|
[
{
"groups": [
"system:masters"
],
"rolearn": "arn:aws:iam::66666666666:role/role1",
"username": "role1"
}
]
| no | -| [map\_users](#input\_map\_users) | Additional IAM users to add to the aws-auth configmap. |
list(object({
userarn = string
username = string
groups = list(string)
}))
|
[
{
"groups": [
"system:masters"
],
"userarn": "arn:aws:iam::66666666666:user/user1",
"username": "user1"
},
{
"groups": [
"system:masters"
],
"userarn": "arn:aws:iam::66666666666:user/user2",
"username": "user2"
}
]
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | -| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | -| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | -| [node\_groups](#output\_node\_groups) | Outputs from node groups | - diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf deleted file mode 100644 index 56a2b05346..0000000000 --- a/examples/managed_node_groups/main.tf +++ /dev/null @@ -1,148 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - name = "managed_node_groups-${random_string.suffix.result}" - cluster_version = "1.20" - region = "eu-west-1" -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_version = local.cluster_version - - vpc_id = module.vpc.vpc_id - subnets = module.vpc.private_subnets - - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - - node_groups_defaults = { - ami_type = "AL2_x86_64" - disk_size = 50 - } - - node_groups = { - example = { - desired_capacity = 1 - max_capacity = 10 - min_capacity = 1 - - instance_types = ["t3.large"] - capacity_type = "SPOT" - k8s_labels = { - Example = "managed_node_groups" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - additional_tags = { - ExtraTag = "example" - } - taints = [ - { - key = "dedicated" - value = "gpuGroup" - effect = "NO_SCHEDULE" - } - ] - update_config = { - max_unavailable_percentage = 50 # or set `max_unavailable` - } - } - example2 = { - desired_capacity = 1 - max_capacity = 10 - min_capacity = 1 - - instance_types = ["t3.medium"] - k8s_labels = { - Example = "managed_node_groups" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - additional_tags = { - ExtraTag = "example2" - } - update_config = { - max_unavailable_percentage = 50 # or set `max_unavailable` - } - } - } - - map_roles = var.map_roles - map_users = var.map_users - map_accounts = var.map_accounts - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# Kubernetes provider configuration -################################################################################ - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} - -################################################################################ -# Supporting Resources -################################################################################ - -data "aws_availability_zones" "available" { -} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} diff --git a/examples/managed_node_groups/outputs.tf b/examples/managed_node_groups/outputs.tf deleted file mode 100644 index 10a3a96604..0000000000 --- a/examples/managed_node_groups/outputs.tf +++ /dev/null @@ -1,24 +0,0 @@ -output "cluster_endpoint" { - description = "Endpoint for EKS control plane." - value = module.eks.cluster_endpoint -} - -output "cluster_security_group_id" { - description = "Security group ids attached to the cluster control plane." - value = module.eks.cluster_security_group_id -} - -output "kubectl_config" { - description = "kubectl config as generated by the module." - value = module.eks.kubeconfig -} - -output "config_map_aws_auth" { - description = "A kubernetes configuration to authenticate to this EKS cluster." - value = module.eks.config_map_aws_auth -} - -output "node_groups" { - description = "Outputs from node groups" - value = module.eks.node_groups -} diff --git a/examples/managed_node_groups/variables.tf b/examples/managed_node_groups/variables.tf deleted file mode 100644 index 57853d8b4d..0000000000 --- a/examples/managed_node_groups/variables.tf +++ /dev/null @@ -1,48 +0,0 @@ -variable "map_accounts" { - description = "Additional AWS account numbers to add to the aws-auth configmap." - type = list(string) - - default = [ - "777777777777", - "888888888888", - ] -} - -variable "map_roles" { - description = "Additional IAM roles to add to the aws-auth configmap." - type = list(object({ - rolearn = string - username = string - groups = list(string) - })) - - default = [ - { - rolearn = "arn:aws:iam::66666666666:role/role1" - username = "role1" - groups = ["system:masters"] - }, - ] -} - -variable "map_users" { - description = "Additional IAM users to add to the aws-auth configmap." - type = list(object({ - userarn = string - username = string - groups = list(string) - })) - - default = [ - { - userarn = "arn:aws:iam::66666666666:user/user1" - username = "user1" - groups = ["system:masters"] - }, - { - userarn = "arn:aws:iam::66666666666:user/user2" - username = "user2" - groups = ["system:masters"] - }, - ] -} diff --git a/examples/managed_node_groups/versions.tf b/examples/managed_node_groups/versions.tf deleted file mode 100644 index 8e2b837984..0000000000 --- a/examples/managed_node_groups/versions.tf +++ /dev/null @@ -1,22 +0,0 @@ -terraform { - required_version = ">= 0.13.1" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.56" - } - local = { - source = "hashicorp/local" - version = ">= 1.4" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 1.11.1" - } - random = { - source = "hashicorp/random" - version = ">= 2.1" - } - } -} diff --git a/examples/secrets_encryption/README.md b/examples/secrets_encryption/README.md deleted file mode 100644 index f5f38b0498..0000000000 --- a/examples/secrets_encryption/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Managed groups example - -This is EKS using [secrets encryption](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) feature. - -See [the official blog](https://aws.amazon.com/blogs/containers/using-eks-encryption-provider-support-for-defense-in-depth/) for more details. - -## Usage - -To run this example you need to execute: - -```bash -$ terraform init -$ terraform plan -$ terraform apply -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [kubernetes](#requirement\_kubernetes) | >= 1.11.1 | -| [local](#requirement\_local) | >= 1.4 | -| [random](#requirement\_random) | >= 2.1 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [random](#provider\_random) | >= 2.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | ../.. | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | -| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | -| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | - -## Inputs - -No inputs. - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. | -| [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. | -| [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. | - diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf deleted file mode 100644 index 49d9a7b029..0000000000 --- a/examples/secrets_encryption/main.tf +++ /dev/null @@ -1,126 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - name = "secrets_encryption-${random_string.suffix.result}" - cluster_version = "1.20" - region = "eu-west-1" -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_version = local.cluster_version - - vpc_id = module.vpc.vpc_id - subnets = module.vpc.private_subnets - - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - - - cluster_encryption_config = [ - { - provider_key_arn = aws_kms_key.eks.arn - resources = ["secrets"] - } - ] - - worker_groups = [ - { - name = "worker-group-1" - instance_type = "t3.small" - additional_userdata = "echo foo bar" - asg_desired_capacity = 2 - }, - ] - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# Kubernetes provider configuration -################################################################################ - -data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id -} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token -} - -################################################################################ -# KMS for encrypting secrets -################################################################################ - -resource "aws_kms_key" "eks" { - description = "EKS Secret Encryption Key" - deletion_window_in_days = 7 - enable_key_rotation = true - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - - -################################################################################ -# Supporting Resources -################################################################################ - -data "aws_availability_zones" "available" { -} - -resource "random_string" "suffix" { - length = 8 - special = false -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = "1" - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" - } - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} diff --git a/examples/secrets_encryption/outputs.tf b/examples/secrets_encryption/outputs.tf deleted file mode 100644 index 359db3a481..0000000000 --- a/examples/secrets_encryption/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -output "cluster_endpoint" { - description = "Endpoint for EKS control plane." - value = module.eks.cluster_endpoint -} - -output "cluster_security_group_id" { - description = "Security group ids attached to the cluster control plane." - value = module.eks.cluster_security_group_id -} - -output "kubectl_config" { - description = "kubectl config as generated by the module." - value = module.eks.kubeconfig -} - -output "config_map_aws_auth" { - description = "A kubernetes configuration to authenticate to this EKS cluster." - value = module.eks.config_map_aws_auth -} diff --git a/examples/secrets_encryption/variables.tf b/examples/secrets_encryption/variables.tf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/secrets_encryption/versions.tf b/examples/secrets_encryption/versions.tf deleted file mode 100644 index 8e2b837984..0000000000 --- a/examples/secrets_encryption/versions.tf +++ /dev/null @@ -1,22 +0,0 @@ -terraform { - required_version = ">= 0.13.1" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.56" - } - local = { - source = "hashicorp/local" - version = ">= 1.4" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 1.11.1" - } - random = { - source = "hashicorp/random" - version = ">= 2.1" - } - } -} diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md new file mode 100644 index 0000000000..ff13d8148a --- /dev/null +++ b/examples/self_managed_node_group/README.md @@ -0,0 +1,95 @@ +# Self Managed Node Groups Example + +Configuration in this directory creates an AWS EKS cluster with various Self Managed Node Groups (AutoScaling Groups) demonstrating the various methods of configuring/customizing: + +- A default, "out of the box" self managed node group as supplied by the `self-managed-node-group` sub-module +- A Bottlerocket self managed node group that demonstrates many of the configuration/customizations offered by the `self-manged-node-group` sub-module for the Bottlerocket OS +- A self managed node group that demonstrates nearly all of the configurations/customizations offered by the `self-managed-node-group` sub-module + +See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for further details. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.64 | +| [null](#requirement\_null) | >= 3.0 | +| [tls](#requirement\_tls) | >= 2.2 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.64 | +| [null](#provider\_null) | >= 3.0 | +| [tls](#provider\_tls) | >= 2.2 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | n/a | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_key_pair.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | +| [aws_kms_key.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [null_resource.apply](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | +| [aws_ami.bottlerocket_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | +| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | +| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | +| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | +| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | +| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | +| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | +| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | +| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | + diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf new file mode 100644 index 0000000000..73b502520c --- /dev/null +++ b/examples/self_managed_node_group/main.tf @@ -0,0 +1,392 @@ +provider "aws" { + region = local.region +} + +locals { + name = "ex-${replace(basename(path.cwd), "_", "-")}" + cluster_version = "1.21" + region = "eu-west-1" + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +data "aws_caller_identity" "current" {} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + cluster_addons = { + coredns = { + resolve_conflicts = "OVERWRITE" + } + kube-proxy = {} + vpc-cni = { + resolve_conflicts = "OVERWRITE" + } + } + + cluster_encryption_config = [{ + provider_key_arn = aws_kms_key.eks.arn + resources = ["secrets"] + }] + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + enable_irsa = true + + self_managed_node_group_defaults = { + disk_size = 50 + } + + self_managed_node_groups = { + # Default node group - as provisioned by the module defaults + default_node_group = {} + + # Bottlerocket node group + bottlerocket = { + name = "bottlerocket-self-mng" + + platform = "bottlerocket" + ami_id = data.aws_ami.bottlerocket_ami.id + instance_type = "m5.large" + desired_size = 2 + key_name = aws_key_pair.this.key_name + + iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"] + + bootstrap_extra_args = <<-EOT + # The admin host container provides SSH access and runs with "superpowers". + # It is disabled by default, but can be disabled explicitly. + [settings.host-containers.admin] + enabled = false + + # The control host container provides out-of-band access via SSM. + # It is enabled by default, and can be disabled if you do not expect to use SSM. + # This could leave you with no way to access the API and change settings on an existing node! + [settings.host-containers.control] + enabled = true + + [settings.kubernetes.node-labels] + ingress = "allowed" + EOT + } + + # Complete + complete = { + name = "complete-self-mng" + use_name_prefix = false + + subnet_ids = module.vpc.public_subnets + + min_size = 1 + max_size = 7 + desired_size = 1 + + ami_id = "ami-0caf35bc73450c396" + bootstrap_extra_args = "--kubelet-extra-args '--max-pods=110'" + + pre_bootstrap_user_data = <<-EOT + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + EOT + + post_bootstrap_user_data = <<-EOT + echo "you are free little kubelet!" + EOT + + disk_size = 256 + instance_type = "m6i.large" + + launch_template_name = "self-managed-ex" + launch_template_use_name_prefix = true + launch_template_description = "Self managed node group example launch template" + + ebs_optimized = true + vpc_security_group_ids = [aws_security_group.additional.id] + enable_monitoring = true + + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = 75 + volume_type = "gp3" + iops = 3000 + throughput = 150 + encrypted = true + kms_key_id = aws_kms_key.ebs.arn + delete_on_termination = true + } + } + } + + metadata_options = { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 2 + } + + create_iam_role = true + iam_role_name = "self-managed-node-group-complete-example" + iam_role_use_name_prefix = false + iam_role_description = "Self managed node group complete example role" + iam_role_tags = { + Purpose = "Protector of the kubelet" + } + iam_role_additional_policies = [ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + ] + + create_security_group = true + security_group_name = "self-managed-node-group-complete-example" + security_group_use_name_prefix = false + security_group_description = "Self managed node group complete example security group" + security_group_rules = { + phoneOut = { + description = "Hello CloudFlare" + protocol = "udp" + from_port = 53 + to_port = 53 + type = "egress" + cidr_blocks = ["1.1.1.1/32"] + } + phoneHome = { + description = "Hello cluster" + protocol = "udp" + from_port = 53 + to_port = 53 + type = "egress" + source_cluster_security_group = true # bit of reflection lookup + } + } + security_group_tags = { + Purpose = "Protector of the kubelet" + } + + timeouts = { + create = "80m" + update = "80m" + delete = "80m" + } + + tags = { + ExtraTag = "Self managed node group complete example" + } + } + } + + tags = local.tags +} + +################################################################################ +# aws-auth configmap +# Only EKS managed node groups automatically add roles to aws-auth configmap +# so we need to ensure fargate profiles and self-managed node roles are added +################################################################################ + +data "aws_eks_cluster_auth" "this" { + name = module.eks.cluster_id +} + +locals { + kubeconfig = yamlencode({ + apiVersion = "v1" + kind = "Config" + current-context = "terraform" + clusters = [{ + name = module.eks.cluster_id + cluster = { + certificate-authority-data = module.eks.cluster_certificate_authority_data + server = module.eks.cluster_endpoint + } + }] + contexts = [{ + name = "terraform" + context = { + cluster = module.eks.cluster_id + user = "terraform" + } + }] + users = [{ + name = "terraform" + user = { + token = data.aws_eks_cluster_auth.this.token + } + }] + }) +} + +resource "null_resource" "apply" { + triggers = { + kubeconfig = base64encode(local.kubeconfig) + cmd_patch = <<-EOT + kubectl create configmap aws-auth -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) + kubectl patch configmap/aws-auth --patch "${module.eks.aws_auth_configmap_yaml}" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) + EOT + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = self.triggers.kubeconfig + } + command = self.triggers.cmd_patch + } +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + + azs = ["${local.region}a", "${local.region}b", "${local.region}c"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + enable_flow_log = true + create_flow_log_cloudwatch_iam_role = true + create_flow_log_cloudwatch_log_group = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} + +resource "aws_security_group" "additional" { + name_prefix = "${local.name}-additional" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + ] + } + + tags = local.tags +} + +resource "aws_kms_key" "eks" { + description = "EKS Secret Encryption Key" + deletion_window_in_days = 7 + enable_key_rotation = true + + tags = local.tags +} + +data "aws_ami" "bottlerocket_ami" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"] + } +} + +resource "tls_private_key" "this" { + algorithm = "RSA" +} + +resource "aws_key_pair" "this" { + key_name = local.name + public_key = tls_private_key.this.public_key_openssh +} + +resource "aws_kms_key" "ebs" { + description = "Customer managed key to encrypt self managed node group volumes" + deletion_window_in_days = 7 + policy = data.aws_iam_policy_document.ebs.json +} + +# This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes +data "aws_iam_policy_document" "ebs" { + # Copy of default KMS policy that lets you manage it + statement { + sid = "Enable IAM User Permissions" + actions = ["kms:*"] + resources = ["*"] + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + } + } + + # Required for EKS + statement { + sid = "Allow service-linked role use of the CMK" + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs + ] + } + } + + statement { + sid = "Allow attachment of persistent resources" + actions = ["kms:CreateGrant"] + resources = ["*"] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs + ] + } + + condition { + test = "Bool" + variable = "kms:GrantIsForAWSResource" + values = ["true"] + } + } +} diff --git a/examples/self_managed_node_group/outputs.tf b/examples/self_managed_node_group/outputs.tf new file mode 100644 index 0000000000..3e9620157b --- /dev/null +++ b/examples/self_managed_node_group/outputs.tf @@ -0,0 +1,167 @@ +################################################################################ +# Cluster +################################################################################ + +output "cluster_arn" { + description = "The Amazon Resource Name (ARN) of the cluster" + value = module.eks.cluster_arn +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded certificate data required to communicate with the cluster" + value = module.eks.cluster_certificate_authority_data +} + +output "cluster_endpoint" { + description = "Endpoint for your Kubernetes API server" + value = module.eks.cluster_endpoint +} + +output "cluster_id" { + description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_id +} + +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_oidc_issuer_url +} + +output "cluster_platform_version" { + description = "Platform version for the cluster" + value = module.eks.cluster_platform_version +} + +output "cluster_status" { + description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" + value = module.eks.cluster_status +} + +output "cluster_primary_security_group_id" { + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = module.eks.cluster_primary_security_group_id +} + +################################################################################ +# Security Group +################################################################################ + +output "cluster_security_group_arn" { + description = "Amazon Resource Name (ARN) of the cluster security group" + value = module.eks.cluster_security_group_arn +} + +output "cluster_security_group_id" { + description = "ID of the cluster security group" + value = module.eks.cluster_security_group_id +} + +################################################################################ +# Node Security Group +################################################################################ + +output "node_security_group_arn" { + description = "Amazon Resource Name (ARN) of the node shared security group" + value = module.eks.node_security_group_arn +} + +output "node_security_group_id" { + description = "ID of the node shared security group" + value = module.eks.node_security_group_id +} + +################################################################################ +# IRSA +################################################################################ + +output "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + value = module.eks.oidc_provider_arn +} + +################################################################################ +# IAM Role +################################################################################ + +output "cluster_iam_role_name" { + description = "IAM role name of the EKS cluster" + value = module.eks.cluster_iam_role_name +} + +output "cluster_iam_role_arn" { + description = "IAM role ARN of the EKS cluster" + value = module.eks.cluster_iam_role_arn +} + +output "cluster_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.cluster_iam_role_unique_id +} + +################################################################################ +# EKS Addons +################################################################################ + +output "cluster_addons" { + description = "Map of attribute maps for all EKS cluster addons enabled" + value = module.eks.cluster_addons +} + +################################################################################ +# EKS Identity Provider +################################################################################ + +output "cluster_identity_providers" { + description = "Map of attribute maps for all EKS identity providers enabled" + value = module.eks.cluster_identity_providers +} + +################################################################################ +# CloudWatch Log Group +################################################################################ + +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_name +} + +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_arn +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.eks.fargate_profiles +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +output "eks_managed_node_groups" { + description = "Map of attribute maps for all EKS managed node groups created" + value = module.eks.eks_managed_node_groups +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +output "self_managed_node_groups" { + description = "Map of attribute maps for all self managed node groups created" + value = module.eks.self_managed_node_groups +} + +################################################################################ +# Additional +################################################################################ + +output "aws_auth_configmap_yaml" { + description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" + value = module.eks.aws_auth_configmap_yaml +} diff --git a/examples/irsa/variables.tf b/examples/self_managed_node_group/variables.tf similarity index 100% rename from examples/irsa/variables.tf rename to examples/self_managed_node_group/variables.tf diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf new file mode 100644 index 0000000000..883963f7b0 --- /dev/null +++ b/examples/self_managed_node_group/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.64" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + tls = { + source = "hashicorp/tls" + version = ">= 2.2" + } + } +} diff --git a/examples/user_data/README.md b/examples/user_data/README.md new file mode 100644 index 0000000000..57ba591944 --- /dev/null +++ b/examples/user_data/README.md @@ -0,0 +1,78 @@ +# Internal User Data Module + +Configuration in this directory render various user data outputs used for testing and validating the internal `_user-data` sub-module. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.64 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks\_mng\_bottlerocket\_additional](#module\_eks\_mng\_bottlerocket\_additional) | ../../modules/_user_data | n/a | +| [eks\_mng\_bottlerocket\_custom\_ami](#module\_eks\_mng\_bottlerocket\_custom\_ami) | ../../modules/_user_data | n/a | +| [eks\_mng\_bottlerocket\_custom\_template](#module\_eks\_mng\_bottlerocket\_custom\_template) | ../../modules/_user_data | n/a | +| [eks\_mng\_bottlerocket\_no\_op](#module\_eks\_mng\_bottlerocket\_no\_op) | ../../modules/_user_data | n/a | +| [eks\_mng\_linux\_additional](#module\_eks\_mng\_linux\_additional) | ../../modules/_user_data | n/a | +| [eks\_mng\_linux\_custom\_ami](#module\_eks\_mng\_linux\_custom\_ami) | ../../modules/_user_data | n/a | +| [eks\_mng\_linux\_custom\_template](#module\_eks\_mng\_linux\_custom\_template) | ../../modules/_user_data | n/a | +| [eks\_mng\_linux\_no\_op](#module\_eks\_mng\_linux\_no\_op) | ../../modules/_user_data | n/a | +| [self\_mng\_bottlerocket\_bootstrap](#module\_self\_mng\_bottlerocket\_bootstrap) | ../../modules/_user_data | n/a | +| [self\_mng\_bottlerocket\_custom\_template](#module\_self\_mng\_bottlerocket\_custom\_template) | ../../modules/_user_data | n/a | +| [self\_mng\_bottlerocket\_no\_op](#module\_self\_mng\_bottlerocket\_no\_op) | ../../modules/_user_data | n/a | +| [self\_mng\_linux\_bootstrap](#module\_self\_mng\_linux\_bootstrap) | ../../modules/_user_data | n/a | +| [self\_mng\_linux\_custom\_template](#module\_self\_mng\_linux\_custom\_template) | ../../modules/_user_data | n/a | +| [self\_mng\_linux\_no\_op](#module\_self\_mng\_linux\_no\_op) | ../../modules/_user_data | n/a | +| [self\_mng\_windows\_bootstrap](#module\_self\_mng\_windows\_bootstrap) | ../../modules/_user_data | n/a | +| [self\_mng\_windows\_custom\_template](#module\_self\_mng\_windows\_custom\_template) | ../../modules/_user_data | n/a | +| [self\_mng\_windows\_no\_op](#module\_self\_mng\_windows\_no\_op) | ../../modules/_user_data | n/a | + +## Resources + +No resources. + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [eks\_mng\_bottlerocket\_additional](#output\_eks\_mng\_bottlerocket\_additional) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_bottlerocket\_custom\_ami](#output\_eks\_mng\_bottlerocket\_custom\_ami) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_bottlerocket\_custom\_template](#output\_eks\_mng\_bottlerocket\_custom\_template) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_bottlerocket\_no\_op](#output\_eks\_mng\_bottlerocket\_no\_op) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_linux\_additional](#output\_eks\_mng\_linux\_additional) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_linux\_custom\_ami](#output\_eks\_mng\_linux\_custom\_ami) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_linux\_custom\_template](#output\_eks\_mng\_linux\_custom\_template) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_linux\_no\_op](#output\_eks\_mng\_linux\_no\_op) | Base64 decoded user data rendered for the provided inputs | +| [self\_mng\_bottlerocket\_bootstrap](#output\_self\_mng\_bottlerocket\_bootstrap) | Base64 decoded user data rendered for the provided inputs | +| [self\_mng\_bottlerocket\_custom\_template](#output\_self\_mng\_bottlerocket\_custom\_template) | Base64 decoded user data rendered for the provided inputs | +| [self\_mng\_bottlerocket\_no\_op](#output\_self\_mng\_bottlerocket\_no\_op) | Base64 decoded user data rendered for the provided inputs | +| [self\_mng\_linux\_bootstrap](#output\_self\_mng\_linux\_bootstrap) | Base64 decoded user data rendered for the provided inputs | +| [self\_mng\_linux\_custom\_template](#output\_self\_mng\_linux\_custom\_template) | Base64 decoded user data rendered for the provided inputs | +| [self\_mng\_linux\_no\_op](#output\_self\_mng\_linux\_no\_op) | Base64 decoded user data rendered for the provided inputs | +| [self\_mng\_windows\_bootstrap](#output\_self\_mng\_windows\_bootstrap) | Base64 decoded user data rendered for the provided inputs | +| [self\_mng\_windows\_custom\_template](#output\_self\_mng\_windows\_custom\_template) | Base64 decoded user data rendered for the provided inputs | +| [self\_mng\_windows\_no\_op](#output\_self\_mng\_windows\_no\_op) | Base64 decoded user data rendered for the provided inputs | + diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf new file mode 100644 index 0000000000..4e961a3947 --- /dev/null +++ b/examples/user_data/main.tf @@ -0,0 +1,289 @@ +locals { + name = "ex-${replace(basename(path.cwd), "_", "-")}" + + cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" + cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" + cluster_service_ipv4_cidr = "172.16.0.0/16" +} + +################################################################################ +# User Data Module +################################################################################ + +# EKS managed node group - linux +module "eks_mng_linux_no_op" { + source = "../../modules/_user_data" +} + +module "eks_mng_linux_additional" { + source = "../../modules/_user_data" + + pre_bootstrap_user_data = <<-EOT + echo "foo" + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + post_bootstrap_user_data = <<-EOT + echo "All done" + EOT +} + +module "eks_mng_linux_custom_ami" { + source = "../../modules/_user_data" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + + enable_bootstrap_user_data = true + + pre_bootstrap_user_data = <<-EOT + echo "foo" + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + post_bootstrap_user_data = <<-EOT + echo "All done" + EOT +} + + +module "eks_mng_linux_custom_template" { + source = "../../modules/_user_data" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + user_data_template_path = "${path.module}/templates/linux_custom.tpl" + + pre_bootstrap_user_data = <<-EOT + echo "foo" + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + post_bootstrap_user_data = <<-EOT + echo "All done" + EOT +} + +# EKS managed node group - bottlerocket +module "eks_mng_bottlerocket_no_op" { + source = "../../modules/_user_data" + + platform = "bottlerocket" +} + +module "eks_mng_bottlerocket_additional" { + source = "../../modules/_user_data" + + platform = "bottlerocket" + + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + EOT +} + +module "eks_mng_bottlerocket_custom_ami" { + source = "../../modules/_user_data" + + platform = "bottlerocket" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + enable_bootstrap_user_data = true + + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + EOT +} + +module "eks_mng_bottlerocket_custom_template" { + source = "../../modules/_user_data" + + platform = "bottlerocket" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + user_data_template_path = "${path.module}/templates/bottlerocket_custom.tpl" + + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + EOT +} + +# Self managed node group - linux +module "self_mng_linux_no_op" { + source = "../../modules/_user_data" + + is_eks_managed_node_group = false +} + +module "self_mng_linux_bootstrap" { + source = "../../modules/_user_data" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + pre_bootstrap_user_data = <<-EOT + echo "foo" + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + post_bootstrap_user_data = <<-EOT + echo "All done" + EOT +} + +module "self_mng_linux_custom_template" { + source = "../../modules/_user_data" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + user_data_template_path = "${path.module}/templates/linux_custom.tpl" + + pre_bootstrap_user_data = <<-EOT + echo "foo" + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + post_bootstrap_user_data = <<-EOT + echo "All done" + EOT +} + +# Self managed node group - bottlerocket +module "self_mng_bottlerocket_no_op" { + source = "../../modules/_user_data" + + platform = "bottlerocket" + + is_eks_managed_node_group = false +} + +module "self_mng_bottlerocket_bootstrap" { + source = "../../modules/_user_data" + + platform = "bottlerocket" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + EOT +} + +module "self_mng_bottlerocket_custom_template" { + source = "../../modules/_user_data" + + platform = "bottlerocket" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + user_data_template_path = "${path.module}/templates/bottlerocket_custom.tpl" + + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + EOT +} + +# Self managed node group - windows +module "self_mng_windows_no_op" { + source = "../../modules/_user_data" + + platform = "windows" + + is_eks_managed_node_group = false +} + +module "self_mng_windows_bootstrap" { + source = "../../modules/_user_data" + + platform = "windows" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + pre_bootstrap_user_data = <<-EOT + [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + EOT + # I don't know if this is the right way on WindowsOS, but its just a string check here anyways + bootstrap_extra_args = "-KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot" + + post_bootstrap_user_data = <<-EOT + [string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + EOT +} + +module "self_mng_windows_custom_template" { + source = "../../modules/_user_data" + + platform = "windows" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + user_data_template_path = "${path.module}/templates/windows_custom.tpl" + + pre_bootstrap_user_data = <<-EOT + [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + EOT + # I don't know if this is the right way on WindowsOS, but its just a string check here anyways + bootstrap_extra_args = "-KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot" + + post_bootstrap_user_data = <<-EOT + [string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + EOT +} diff --git a/examples/user_data/outputs.tf b/examples/user_data/outputs.tf new file mode 100644 index 0000000000..dd2c3407e1 --- /dev/null +++ b/examples/user_data/outputs.tf @@ -0,0 +1,89 @@ +# EKS managed node group - linux +output "eks_mng_linux_no_op" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_linux_no_op.user_data) +} + +output "eks_mng_linux_additional" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_linux_additional.user_data) +} + +output "eks_mng_linux_custom_ami" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_linux_custom_ami.user_data) +} + +output "eks_mng_linux_custom_template" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_linux_custom_template.user_data) +} + +# EKS managed node group - bottlerocket +output "eks_mng_bottlerocket_no_op" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_bottlerocket_no_op.user_data) +} + +output "eks_mng_bottlerocket_additional" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_bottlerocket_additional.user_data) +} + +output "eks_mng_bottlerocket_custom_ami" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_bottlerocket_custom_ami.user_data) +} + +output "eks_mng_bottlerocket_custom_template" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_bottlerocket_custom_template.user_data) +} + +# Self managed node group - linux +output "self_mng_linux_no_op" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.self_mng_linux_no_op.user_data) +} + +output "self_mng_linux_bootstrap" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.self_mng_linux_bootstrap.user_data) +} + +output "self_mng_linux_custom_template" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.self_mng_linux_custom_template.user_data) +} + +# Self managed node group - bottlerocket +output "self_mng_bottlerocket_no_op" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.self_mng_bottlerocket_no_op.user_data) +} + +output "self_mng_bottlerocket_bootstrap" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.self_mng_bottlerocket_bootstrap.user_data) +} + +output "self_mng_bottlerocket_custom_template" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.self_mng_bottlerocket_custom_template.user_data) +} + +# Self managed node group - windows +output "self_mng_windows_no_op" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.self_mng_windows_no_op.user_data) +} + +output "self_mng_windows_bootstrap" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.self_mng_windows_bootstrap.user_data) +} + +output "self_mng_windows_custom_template" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.self_mng_windows_custom_template.user_data) +} diff --git a/examples/user_data/templates/bottlerocket_custom.tpl b/examples/user_data/templates/bottlerocket_custom.tpl new file mode 100644 index 0000000000..6c4d9434a7 --- /dev/null +++ b/examples/user_data/templates/bottlerocket_custom.tpl @@ -0,0 +1,7 @@ +# Custom user data template provided for rendering +[settings.kubernetes] +"cluster-name" = "${cluster_name}" +"api-server" = "${cluster_endpoint}" +"cluster-certificate" = "${cluster_auth_base64}" + +${bootstrap_extra_args~} diff --git a/examples/user_data/templates/linux_custom.tpl b/examples/user_data/templates/linux_custom.tpl new file mode 100644 index 0000000000..bfe21f117a --- /dev/null +++ b/examples/user_data/templates/linux_custom.tpl @@ -0,0 +1,10 @@ +#!/bin/bash +set -ex + +${pre_bootstrap_user_data ~} + +# Custom user data template provided for rendering +B64_CLUSTER_CA=${cluster_auth_base64} +API_SERVER_URL=${cluster_endpoint} +/etc/eks/bootstrap.sh ${cluster_name} ${bootstrap_extra_args} --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +${post_bootstrap_user_data ~} diff --git a/examples/user_data/templates/windows_custom.tpl b/examples/user_data/templates/windows_custom.tpl new file mode 100644 index 0000000000..3c1ca7014a --- /dev/null +++ b/examples/user_data/templates/windows_custom.tpl @@ -0,0 +1,10 @@ +# Custom user data template provided for rendering + +${pre_bootstrap_user_data ~} +[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" +[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' +[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" +& $EKSBootstrapScriptFile -EKSClusterName ${cluster_name} -APIServerEndpoint ${cluster_endpoint} -Base64ClusterCA ${cluster_auth_base64} ${bootstrap_extra_args} 3>&1 4>&1 5>&1 6>&1 +$LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } +${post_bootstrap_user_data ~} + diff --git a/examples/launch_templates/variables.tf b/examples/user_data/variables.tf similarity index 100% rename from examples/launch_templates/variables.tf rename to examples/user_data/variables.tf diff --git a/examples/user_data/versions.tf b/examples/user_data/versions.tf new file mode 100644 index 0000000000..bfce6ae345 --- /dev/null +++ b/examples/user_data/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.64" + } + } +} diff --git a/fargate.tf b/fargate.tf deleted file mode 100644 index 5526e2eed0..0000000000 --- a/fargate.tf +++ /dev/null @@ -1,16 +0,0 @@ -module "fargate" { - source = "./modules/fargate" - - create_eks = var.create_eks - create_fargate_pod_execution_role = var.create_fargate_pod_execution_role - - cluster_name = local.cluster_name - fargate_pod_execution_role_name = var.fargate_pod_execution_role_name - permissions_boundary = var.permissions_boundary - iam_path = var.iam_path - subnets = coalescelist(var.fargate_subnets, var.subnets, [""]) - - fargate_profiles = var.fargate_profiles - - tags = var.tags -} diff --git a/irsa.tf b/irsa.tf deleted file mode 100644 index 5fc3dc8df7..0000000000 --- a/irsa.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Enable IAM Roles for EKS Service-Accounts (IRSA). - -# The Root CA Thumbprint for an OpenID Connect Identity Provider is currently -# Being passed as a default value which is the same for all regions and -# Is valid until (Jun 28 17:39:16 2034 GMT). -# https://crt.sh/?q=9E99A48A9960B14926BB7F3B02E22DA2B0AB7280 -# https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc_verify-thumbprint.html -# https://github.com/terraform-providers/terraform-provider-aws/issues/10104 - -resource "aws_iam_openid_connect_provider" "oidc_provider" { - count = var.enable_irsa && var.create_eks ? 1 : 0 - - client_id_list = local.client_id_list - thumbprint_list = [var.eks_oidc_root_ca_thumbprint] - url = local.cluster_oidc_issuer_url - - tags = merge( - { - Name = "${var.cluster_name}-eks-irsa" - }, - var.tags - ) -} diff --git a/kubectl.tf b/kubectl.tf deleted file mode 100644 index b5d6947855..0000000000 --- a/kubectl.tf +++ /dev/null @@ -1,8 +0,0 @@ -resource "local_file" "kubeconfig" { - count = var.write_kubeconfig && var.create_eks ? 1 : 0 - - content = local.kubeconfig - filename = substr(var.kubeconfig_output_path, -1, 1) == "/" ? "${var.kubeconfig_output_path}kubeconfig_${var.cluster_name}" : var.kubeconfig_output_path - file_permission = var.kubeconfig_file_permission - directory_permission = "0755" -} diff --git a/locals.tf b/locals.tf deleted file mode 100644 index 8e1b54e49c..0000000000 --- a/locals.tf +++ /dev/null @@ -1,263 +0,0 @@ -locals { - - # EKS Cluster - cluster_id = coalescelist(aws_eks_cluster.this[*].id, [""])[0] - cluster_arn = coalescelist(aws_eks_cluster.this[*].arn, [""])[0] - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] - cluster_endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] - cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] - cluster_oidc_issuer_url = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc[0].issuer, [""]))[0] - cluster_primary_security_group_id = coalescelist(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, [""])[0] - - cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id - cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name - cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) - - # Worker groups - worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id - - default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] - default_ami_id_linux = local.workers_group_defaults.ami_id != "" ? local.workers_group_defaults.ami_id : concat(data.aws_ami.eks_worker.*.id, [""])[0] - default_ami_id_windows = local.workers_group_defaults.ami_id_windows != "" ? local.workers_group_defaults.ami_id_windows : concat(data.aws_ami.eks_worker_windows.*.id, [""])[0] - - worker_group_launch_configuration_count = length(var.worker_groups) - worker_group_launch_template_count = length(var.worker_groups_launch_template) - - worker_groups_platforms = [for x in concat(var.worker_groups, var.worker_groups_launch_template) : try(x.platform, var.workers_group_defaults["platform"], var.default_platform)] - - worker_ami_name_filter = coalesce(var.worker_ami_name_filter, "amazon-eks-node-${coalesce(var.cluster_version, "cluster_version")}-v*") - worker_ami_name_filter_windows = coalesce(var.worker_ami_name_filter_windows, "Windows_Server-2019-English-Core-EKS_Optimized-${coalesce(var.cluster_version, "cluster_version")}-*") - - ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" - sts_principal = "sts.${data.aws_partition.current.dns_suffix}" - client_id_list = distinct(compact(concat([local.sts_principal], var.openid_connect_audiences))) - policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" - - workers_group_defaults_defaults = { - name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. - tags = [] # A list of maps defining extra tags to be applied to the worker group autoscaling group and volumes. - ami_id = "" # AMI ID for the eks linux based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. - ami_id_windows = "" # AMI ID for the eks windows based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. - asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desired_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. - asg_max_size = "3" # Maximum worker capacity in the autoscaling group. - asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. - asg_force_delete = false # Enable forced deletion for the autoscaling group. - asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. - default_cooldown = null # The amount of time, in seconds, after a scaling activity completes before another scaling activity can start. - health_check_type = null # Controls how health checking is done. Valid values are "EC2" or "ELB". - health_check_grace_period = null # Time in seconds after instance comes into service before checking health. - instance_type = "m4.large" # Size of the workers instances. - instance_store_virtual_name = "ephemeral0" # "virtual_name" of the instance store volume. - spot_price = "" # Cost of spot instance. - placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". - root_volume_size = "100" # root volume size of workers instances. - root_volume_type = "gp2" # root volume type of workers instances, can be "standard", "gp3", "gp2", or "io1" - root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". - root_volume_throughput = null # The amount of throughput to provision for a gp3 volume. - key_name = "" # The key pair name that should be used for the instances in the autoscaling group - pre_userdata = "" # userdata to pre-append to the default userdata. - userdata_template_file = "" # alternate template to use for userdata - userdata_template_extra_args = {} # Additional arguments to use when expanding the userdata template file - bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI (Amazon Machine Image). - additional_userdata = "" # userdata to append to the default userdata. - ebs_optimized = true # sets whether to use ebs optimization on supported types. - enable_monitoring = true # Enables/disables detailed monitoring. - enclave_support = false # Enables/disables enclave support - public_ip = false # Associate a public ip address with a worker - kubelet_extra_args = "" # This string is passed directly to kubelet if set. Useful for adding labels or taints. - subnets = var.subnets # A list of subnets to place the worker nodes in. i.e. ["subnet-123", "subnet-456", "subnet-789"] - additional_security_group_ids = [] # A list of additional security group ids to include in worker launch config - protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. - iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. - iam_role_id = "local.default_iam_role_id" # A custom IAM role id. Incompatible with iam_instance_profile_name. Literal local.default_iam_role_id will never be used but if iam_role_id is not set, the local.default_iam_role_id interpolation will be used. - suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] - target_group_arns = null # A list of Application LoadBalancer (ALB) target group ARNs to be associated to the autoscaling group - load_balancers = null # A list of Classic LoadBalancer (CLB)'s name to be associated to the autoscaling group - enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] - placement_group = null # The name of the placement group into which to launch the instances, if any. - service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS - termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. - platform = var.default_platform # Platform of workers. Either "linux" or "windows". - additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, throughput, encrypted, kms_key_id (only on launch-template), delete_on_termination, snapshot_id. Optional values are grabbed from root volume or from defaults - additional_instance_store_volumes = [] # A list of additional instance store (local disk) volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), virtual_name. - warm_pool = null # If this block is configured, add a Warm Pool to the specified Auto Scaling group. - timeouts = {} # A map of timeouts for create/update/delete operations - snapshot_id = null # A custom snapshot ID. - - # Settings for launch templates - root_block_device_name = concat(data.aws_ami.eks_worker.*.root_device_name, [""])[0] # Root device name for Linux workers. If not provided, will assume default Linux AMI was used. - root_block_device_name_windows = concat(data.aws_ami.eks_worker_windows.*.root_device_name, [""])[0] # Root device name for Windows workers. If not provided, will assume default Windows AMI was used. - root_kms_key_id = "" # The KMS key to use when encrypting the root storage device - launch_template_id = null # The id of the launch template used for managed node_groups - launch_template_version = "$Latest" # The latest version of the launch template to use in the autoscaling group - update_default_version = false # Update the autoscaling group launch template's default version upon each update - launch_template_placement_tenancy = "default" # The placement tenancy for instances - launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. - root_encrypted = false # Whether the volume should be encrypted or not - eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) - interface_type = null # The type of network interface. To create an Elastic Fabric Adapter (EFA), specify 'efa'. - cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs - market_type = null - metadata_http_endpoint = "enabled" # The state of the metadata service: enabled, disabled. - metadata_http_tokens = "optional" # If session tokens are required: optional, required. - metadata_http_put_response_hop_limit = null # The desired HTTP PUT response hop limit for instance metadata requests. - # Settings for launch templates with mixed instances policy - override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] # A list of override instance types for mixed instances policy - on_demand_allocation_strategy = null # Strategy to use when launching on-demand instances. Valid values: prioritized. - on_demand_base_capacity = "0" # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances - on_demand_percentage_above_base_capacity = "0" # Percentage split between on-demand and Spot instances above the base on-demand capacity - spot_allocation_strategy = "lowest-price" # Valid options are 'lowest-price' and 'capacity-optimized'. If 'lowest-price', the Auto Scaling group launches instances using the Spot pools with the lowest price, and evenly allocates your instances across the number of Spot pools. If 'capacity-optimized', the Auto Scaling group launches instances using Spot pools that are optimally chosen based on the available Spot capacity. - spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." - spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price - max_instance_lifetime = 0 # Maximum number of seconds instances can run in the ASG. 0 is unlimited. - elastic_inference_accelerator = null # Type of elastic inference accelerator to be attached. Example values are eia1.medium, eia2.large, etc. - instance_refresh_enabled = false # Enable instance refresh for the worker autoscaling group. - instance_refresh_strategy = "Rolling" # Strategy to use for instance refresh. Default is 'Rolling' which the only valid value. - instance_refresh_min_healthy_percentage = 90 # The amount of capacity in the ASG that must remain healthy during an instance refresh, as a percentage of the ASG's desired capacity. - instance_refresh_instance_warmup = null # The number of seconds until a newly launched instance is configured and ready to use. Defaults to the ASG's health check grace period. - instance_refresh_triggers = [] # Set of additional property names that will trigger an Instance Refresh. A refresh will always be triggered by a change in any of launch_configuration, launch_template, or mixed_instances_policy. - capacity_rebalance = false # Enable capacity rebalance - } - - workers_group_defaults = merge( - local.workers_group_defaults_defaults, - var.workers_group_defaults, - ) - - ebs_optimized_not_supported = [ - "c1.medium", - "c3.8xlarge", - "c3.large", - "c5d.12xlarge", - "c5d.24xlarge", - "c5d.metal", - "cc2.8xlarge", - "cr1.8xlarge", - "g2.8xlarge", - "g4dn.metal", - "hs1.8xlarge", - "i2.8xlarge", - "m1.medium", - "m1.small", - "m2.xlarge", - "m3.large", - "m3.medium", - "m5ad.16xlarge", - "m5ad.8xlarge", - "m5dn.metal", - "m5n.metal", - "r3.8xlarge", - "r3.large", - "r5ad.16xlarge", - "r5ad.8xlarge", - "r5dn.metal", - "r5n.metal", - "t1.micro", - "t2.2xlarge", - "t2.large", - "t2.medium", - "t2.micro", - "t2.nano", - "t2.small", - "t2.xlarge" - ] - - kubeconfig = var.create_eks ? templatefile("${path.module}/templates/kubeconfig.tpl", { - kubeconfig_name = coalesce(var.kubeconfig_name, "eks_${var.cluster_name}") - endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 - aws_authenticator_kubeconfig_apiversion = var.kubeconfig_api_version - aws_authenticator_command = var.kubeconfig_aws_authenticator_command - aws_authenticator_command_args = coalescelist(var.kubeconfig_aws_authenticator_command_args, ["token", "-i", local.cluster_name]) - aws_authenticator_additional_args = var.kubeconfig_aws_authenticator_additional_args - aws_authenticator_env_variables = var.kubeconfig_aws_authenticator_env_variables - }) : "" - - launch_configuration_userdata_rendered = [ - for index in range(var.create_eks ? local.worker_group_launch_configuration_count : 0) : templatefile( - lookup( - var.worker_groups[index], - "userdata_template_file", - lookup(var.worker_groups[index], "platform", local.workers_group_defaults["platform"]) == "windows" - ? "${path.module}/templates/userdata_windows.tpl" - : "${path.module}/templates/userdata.sh.tpl" - ), - merge({ - platform = lookup(var.worker_groups[index], "platform", local.workers_group_defaults["platform"]) - cluster_name = local.cluster_name - endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 - pre_userdata = lookup( - var.worker_groups[index], - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - var.worker_groups[index], - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - var.worker_groups[index], - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - var.worker_groups[index], - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - var.worker_groups[index], - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) - ) - ] - - launch_template_userdata_rendered = [ - for index in range(var.create_eks ? local.worker_group_launch_template_count : 0) : templatefile( - lookup( - var.worker_groups_launch_template[index], - "userdata_template_file", - lookup(var.worker_groups_launch_template[index], "platform", local.workers_group_defaults["platform"]) == "windows" - ? "${path.module}/templates/userdata_windows.tpl" - : "${path.module}/templates/userdata.sh.tpl" - ), - merge({ - platform = lookup(var.worker_groups_launch_template[index], "platform", local.workers_group_defaults["platform"]) - cluster_name = local.cluster_name - endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 - pre_userdata = lookup( - var.worker_groups_launch_template[index], - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - var.worker_groups_launch_template[index], - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - var.worker_groups_launch_template[index], - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - var.worker_groups_launch_template[index], - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - var.worker_groups_launch_template[index], - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) - ) - ] -} diff --git a/main.tf b/main.tf index d1860f0696..286e23b5a4 100644 --- a/main.tf +++ b/main.tf @@ -1,24 +1,20 @@ -resource "aws_cloudwatch_log_group" "this" { - count = length(var.cluster_enabled_log_types) > 0 && var.create_eks ? 1 : 0 +data "aws_partition" "current" {} - name = "/aws/eks/${var.cluster_name}/cluster" - retention_in_days = var.cluster_log_retention_in_days - kms_key_id = var.cluster_log_kms_key_id - - tags = var.tags -} +################################################################################ +# Cluster +################################################################################ resource "aws_eks_cluster" "this" { - count = var.create_eks ? 1 : 0 + count = var.create ? 1 : 0 name = var.cluster_name - enabled_cluster_log_types = var.cluster_enabled_log_types - role_arn = local.cluster_iam_role_arn + role_arn = try(aws_iam_role.this[0].arn, var.iam_role_arn) version = var.cluster_version + enabled_cluster_log_types = var.cluster_enabled_log_types vpc_config { - security_group_ids = compact([local.cluster_security_group_id]) - subnet_ids = var.subnets + security_group_ids = distinct(concat(var.cluster_additional_security_group_ids, [local.cluster_security_group_id])) + subnet_ids = var.subnet_ids endpoint_private_access = var.cluster_endpoint_private_access endpoint_public_access = var.cluster_endpoint_public_access public_access_cidrs = var.cluster_endpoint_public_access_cidrs @@ -45,188 +41,221 @@ resource "aws_eks_cluster" "this" { ) timeouts { - create = var.cluster_create_timeout - delete = var.cluster_delete_timeout - update = var.cluster_update_timeout + create = lookup(var.cluster_timeouts, "create", null) + delete = lookup(var.cluster_timeouts, "update", null) + update = lookup(var.cluster_timeouts, "delete", null) } depends_on = [ - aws_security_group_rule.cluster_egress_internet, - aws_security_group_rule.cluster_https_worker_ingress, - aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy, - aws_iam_role_policy_attachment.cluster_AmazonEKSServicePolicy, - aws_iam_role_policy_attachment.cluster_AmazonEKSVPCResourceControllerPolicy, + aws_iam_role_policy_attachment.this, + aws_security_group_rule.cluster, + aws_security_group_rule.node, aws_cloudwatch_log_group.this ] } +resource "aws_cloudwatch_log_group" "this" { + count = var.create && var.create_cloudwatch_log_group ? 1 : 0 + + name = "/aws/eks/${var.cluster_name}/cluster" + retention_in_days = var.cloudwatch_log_group_retention_in_days + kms_key_id = var.cloudwatch_log_group_kms_key_id + + tags = var.tags +} + +################################################################################ +# Cluster Security Group +# Defaults follow https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html +################################################################################ + +locals { + cluster_sg_name = coalesce(var.cluster_security_group_name, "${var.cluster_name}-cluster") + create_cluster_sg = var.create && var.create_cluster_security_group + + cluster_security_group_id = local.create_cluster_sg ? aws_security_group.cluster[0].id : var.cluster_security_group_id + + cluster_security_group_rules = { + ingress_nodes_443 = { + description = "Node groups to cluster API" + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "ingress" + source_node_security_group = true + } + egress_nodes_443 = { + description = "Cluster API to node groups" + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "egress" + source_node_security_group = true + } + egress_nodes_kubelet = { + description = "Cluster API to node kubelets" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + type = "egress" + source_node_security_group = true + } + } +} + resource "aws_security_group" "cluster" { - count = var.cluster_create_security_group && var.create_eks ? 1 : 0 + count = local.create_cluster_sg ? 1 : 0 - name_prefix = var.cluster_name - description = "EKS cluster security group." + name = var.cluster_security_group_use_name_prefix ? null : local.cluster_sg_name + name_prefix = var.cluster_security_group_use_name_prefix ? "${local.cluster_sg_name}-" : null + description = var.cluster_security_group_description vpc_id = var.vpc_id tags = merge( var.tags, - { - "Name" = "${var.cluster_name}-eks_cluster_sg" - }, + { "Name" = local.cluster_sg_name }, + var.cluster_security_group_tags ) } -resource "aws_security_group_rule" "cluster_egress_internet" { - count = var.cluster_create_security_group && var.create_eks ? 1 : 0 - - description = "Allow cluster egress access to the Internet." - protocol = "-1" - security_group_id = local.cluster_security_group_id - cidr_blocks = var.cluster_egress_cidrs - from_port = 0 - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "cluster_https_worker_ingress" { - count = var.cluster_create_security_group && var.create_eks && var.worker_create_security_group ? 1 : 0 - - description = "Allow pods to communicate with the EKS cluster API." - protocol = "tcp" - security_group_id = local.cluster_security_group_id - source_security_group_id = local.worker_security_group_id - from_port = 443 - to_port = 443 - type = "ingress" +resource "aws_security_group_rule" "cluster" { + for_each = local.create_cluster_sg ? merge(local.cluster_security_group_rules, var.cluster_security_group_additional_rules) : {} + + # Required + security_group_id = aws_security_group.cluster[0].id + protocol = each.value.protocol + from_port = each.value.from_port + to_port = each.value.to_port + type = each.value.type + + # Optional + description = try(each.value.description, null) + cidr_blocks = try(each.value.cidr_blocks, null) + ipv6_cidr_blocks = try(each.value.ipv6_cidr_blocks, null) + prefix_list_ids = try(each.value.prefix_list_ids, []) + self = try(each.value.self, null) + source_security_group_id = try( + each.value.source_security_group_id, + try(each.value.source_node_security_group, false) ? local.node_security_group_id : null + ) } -resource "aws_security_group_rule" "cluster_private_access_cidrs_source" { - for_each = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_cidrs != null ? toset(var.cluster_endpoint_private_access_cidrs) : [] +################################################################################ +# IRSA +# Note - this is different from EKS identity provider +################################################################################ - description = "Allow private K8S API ingress from custom CIDR source." - type = "ingress" - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = [each.value] +data "tls_certificate" "this" { + count = var.create && var.enable_irsa ? 1 : 0 - security_group_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id + url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer } -resource "aws_security_group_rule" "cluster_private_access_sg_source" { - count = var.create_eks && var.cluster_create_endpoint_private_access_sg_rule && var.cluster_endpoint_private_access && var.cluster_endpoint_private_access_sg != null ? length(var.cluster_endpoint_private_access_sg) : 0 +resource "aws_iam_openid_connect_provider" "oidc_provider" { + count = var.create && var.enable_irsa ? 1 : 0 - description = "Allow private K8S API ingress from custom Security Groups source." - type = "ingress" - from_port = 443 - to_port = 443 - protocol = "tcp" - source_security_group_id = var.cluster_endpoint_private_access_sg[count.index] + client_id_list = distinct(compact(concat(["sts.${data.aws_partition.current.dns_suffix}"], var.openid_connect_audiences))) + thumbprint_list = [data.tls_certificate.this[0].certificates[0].sha1_fingerprint] + url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer - security_group_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id + tags = merge( + { Name = "${var.cluster_name}-eks-irsa" }, + var.tags + ) } -resource "aws_iam_role" "cluster" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 - - name_prefix = var.cluster_iam_role_name != "" ? null : var.cluster_name - name = var.cluster_iam_role_name != "" ? var.cluster_iam_role_name : null - assume_role_policy = data.aws_iam_policy_document.cluster_assume_role_policy.json - permissions_boundary = var.permissions_boundary - path = var.iam_path - force_detach_policies = true +################################################################################ +# IAM Role +################################################################################ - tags = var.tags +locals { + iam_role_name = coalesce(var.iam_role_name, "${var.cluster_name}-cluster") + policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" } -resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 - - policy_arn = "${local.policy_arn_prefix}/AmazonEKSClusterPolicy" - role = local.cluster_iam_role_name -} +data "aws_iam_policy_document" "assume_role_policy" { + count = var.create && var.create_iam_role ? 1 : 0 -resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + statement { + sid = "EKSClusterAssumeRole" + actions = ["sts:AssumeRole"] - policy_arn = "${local.policy_arn_prefix}/AmazonEKSServicePolicy" - role = local.cluster_iam_role_name + principals { + type = "Service" + identifiers = ["eks.amazonaws.com"] + } + } } -resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSVPCResourceControllerPolicy" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 +resource "aws_iam_role" "this" { + count = var.create && var.create_iam_role ? 1 : 0 - policy_arn = "${local.policy_arn_prefix}/AmazonEKSVPCResourceController" - role = local.cluster_iam_role_name -} - -/* - Adding a policy to cluster IAM role that allow permissions - required to create AWSServiceRoleForElasticLoadBalancing service-linked role by EKS during ELB provisioning -*/ + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description -data "aws_iam_policy_document" "cluster_elb_sl_role_creation" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + assume_role_policy = data.aws_iam_policy_document.assume_role_policy[0].json + permissions_boundary = var.iam_role_permissions_boundary + force_detach_policies = true - statement { - effect = "Allow" - actions = [ - "ec2:DescribeAccountAttributes", - "ec2:DescribeInternetGateways", - "ec2:DescribeAddresses" - ] - resources = ["*"] - } + tags = merge(var.tags, var.iam_role_tags) } -resource "aws_iam_policy" "cluster_elb_sl_role_creation" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 +# Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group +resource "aws_iam_role_policy_attachment" "this" { + for_each = var.create && var.create_iam_role ? toset(compact(distinct(concat([ + "${local.policy_arn_prefix}/AmazonEKSClusterPolicy", + "${local.policy_arn_prefix}/AmazonEKSVPCResourceController", + ], var.iam_role_additional_policies)))) : toset([]) - name_prefix = "${var.cluster_name}-elb-sl-role-creation" - description = "Permissions for EKS to create AWSServiceRoleForElasticLoadBalancing service-linked role" - policy = data.aws_iam_policy_document.cluster_elb_sl_role_creation[0].json - path = var.iam_path - - tags = var.tags + policy_arn = each.value + role = aws_iam_role.this[0].name } -resource "aws_iam_role_policy_attachment" "cluster_elb_sl_role_creation" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 +################################################################################ +# EKS Addons +################################################################################ - policy_arn = aws_iam_policy.cluster_elb_sl_role_creation[0].arn - role = local.cluster_iam_role_name -} +resource "aws_eks_addon" "this" { + for_each = { for k, v in var.cluster_addons : k => v if var.create } -/* - Adding a policy to cluster IAM role that deny permissions to logs:CreateLogGroup - it is not needed since we create the log group ourselve in this module, and it is causing trouble during cleanup/deletion -*/ + cluster_name = aws_eks_cluster.this[0].name + addon_name = try(each.value.name, each.key) -data "aws_iam_policy_document" "cluster_deny_log_group" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 + addon_version = lookup(each.value, "addon_version", null) + resolve_conflicts = lookup(each.value, "resolve_conflicts", null) + service_account_role_arn = lookup(each.value, "service_account_role_arn", null) - statement { - effect = "Deny" - actions = [ - "logs:CreateLogGroup" + lifecycle { + ignore_changes = [ + modified_at ] - resources = ["*"] } -} - -resource "aws_iam_policy" "cluster_deny_log_group" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 - - name_prefix = "${var.cluster_name}-deny-log-group" - description = "Deny CreateLogGroup" - policy = data.aws_iam_policy_document.cluster_deny_log_group[0].json - path = var.iam_path tags = var.tags } -resource "aws_iam_role_policy_attachment" "cluster_deny_log_group" { - count = var.manage_cluster_iam_resources && var.create_eks ? 1 : 0 +################################################################################ +# EKS Identity Provider +# Note - this is different from IRSA +################################################################################ + +resource "aws_eks_identity_provider_config" "this" { + for_each = { for k, v in var.cluster_identity_providers : k => v if var.create } + + cluster_name = aws_eks_cluster.this[0].name + + oidc { + client_id = each.value.client_id + groups_claim = lookup(each.value, "groups_claim", null) + groups_prefix = lookup(each.value, "groups_prefix", null) + identity_provider_config_name = try(each.value.identity_provider_config_name, each.key) + issuer_url = each.value.issuer_url + required_claims = lookup(each.value, "required_claims", null) + username_claim = lookup(each.value, "username_claim", null) + username_prefix = lookup(each.value, "username_prefix", null) + } - policy_arn = aws_iam_policy.cluster_deny_log_group[0].arn - role = local.cluster_iam_role_name + tags = var.tags } diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md new file mode 100644 index 0000000000..f61500517e --- /dev/null +++ b/modules/_user_data/README.md @@ -0,0 +1,123 @@ +# Internal User Data Module + +Configuration in this directory renders the appropriate user data for the given inputs. There are a number of different ways that user data can be utilized and this internal module is designed to aid in making that flexibility possible as well as providing a means for out of bands testing and validation. + +See the [`examples/user_data/` directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) for various examples of using the module. + +## Combinations + +At a high level, AWS EKS users have two methods for launching nodes within this EKS module (ignoring Fargate profiles): + +1. EKS managed node group +2. Self managed node group + +### EKS Managed Node Group + +When using an EKS managed node group, users have 2 primary routes for interacting with the bootstrap user data: + +1. If the EKS managed node group does **NOT** utilize a custom AMI, then users can elect to supply additional user data that is pre-pended before the EKS managed node group bootstrap user data. You can read more about this process from the [AWS supplied documentation](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data) + + - Users can use the following variables to facilitate this process: + + ```hcl + pre_bootstrap_user_data = "..." + bootstrap_extra_args = "..." + ``` + +2. If the EKS managed node group does utilize a custom AMI, then per the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami), users will need to supply the necessary bootstrap configuration via user data to ensure that the node is configured to register with the cluster when launched. There are two routes that users can utilize to facilitate this bootstrapping process: + - If the AMI used is a derivative of the [AWS EKS Optimized AMI ](https://github.com/awslabs/amazon-eks-ami), users can opt in to using a template provided by the module that provides the minimum necessary configuration to bootstrap the node when launched, with the option to add additional pre and post bootstrap user data as well as bootstrap additional args that are supplied to the [AWS EKS bootstrap.sh script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh) + - Users can use the following variables to facilitate this process: + ```hcl + enable_bootstrap_user_data = true # to opt in to using the module supplied bootstrap user data template + pre_bootstrap_user_data = "..." + bootstrap_extra_args = "..." + post_bootstrap_user_data = "..." + ``` + - If the AMI is not an AWS EKS Optimized AMI derivative, or if users wish to have more control over the user data that is supplied to the node when launched, users have the ability to supply their own user data template that will be rendered instead of the module supplied template. Note - only the variables that are supplied to the `templatefile()` for the respective platform/OS are available for use in the supplied template, otherwise users will need to pre-render/pre-populate the template before supplying the final template to the module for rendering as user data. + - Users can use the following variables to facilitate this process: + ```hcl + user_data_template_path = "./your/user_data.sh" # user supplied bootstrap user data template + pre_bootstrap_user_data = "..." + bootstrap_extra_args = "..." + post_bootstrap_user_data = "..." + ``` + +| ℹ️ When using bottlerocket as the desired platform, since the user data for bottlerocket is TOML, all configurations are merged in the one file supplied as user data. Therefore, `pre_bootstrap_user_data` and `post_bootstrap_user_data` are not valid since the bottlerocket OS handles when various settings are applied. If you wish to supply additional configuration settings when using bottlerocket, supply them via the `bootstrap_extra_args` variable. For the linux platform, `bootstrap_extra_args` are settings that will be supplied to the [AWS EKS Optimized AMI bootstrap script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh#L14) such as kubelet extra args, etc. See the [bottlerocket GitHub repository documentation](https://github.com/bottlerocket-os/bottlerocket#description-of-settings) for more details on what settings can be supplied via the `bootstrap_extra_args` variable. | +| :--- | + +### Self Managed Node Group + +When using a self managed node group, the options presented to users is very similar to the 2nd option listed above for EKS managed node groups. Since self managed node groups require users to provide the bootstrap user data, there is no concept of appending to user data that AWS provides; users can either elect to use the user data template provided for their platform/OS by the module or provide their own user data template for rendering by the module. + +- If the AMI used is a derivative of the [AWS EKS Optimized AMI ](https://github.com/awslabs/amazon-eks-ami), users can opt in to using a template provided by the module that provides the minimum necessary configuration to bootstrap the node when launched, with the option to add additional pre and post bootstrap user data as well as bootstrap additional args that are supplied to the [AWS EKS bootstrap.sh script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh) + - Users can use the following variables to facilitate this process: + ```hcl + enable_bootstrap_user_data = true # to opt in to using the module supplied bootstrap user data template + pre_bootstrap_user_data = "..." + bootstrap_extra_args = "..." + post_bootstrap_user_data = "..." + ``` +- If the AMI is not an AWS EKS Optimized AMI derivative, or if users wish to have more control over the user data that is supplied to the node upon launch, users have the ability to supply their own user data template that will be rendered instead of the module supplied template. Note - only the variables that are supplied to the `templatefile()` for the respective platform/OS are available for use in the supplied template, otherwise users will need to pre-render/pre-populate the template before supplying the final template to the module for rendering as user data. + - Users can use the following variables to facilitate this process: + ```hcl + user_data_template_path = "./your/user_data.sh" # user supplied bootstrap user data template + pre_bootstrap_user_data = "..." + bootstrap_extra_args = "..." + post_bootstrap_user_data = "..." + ``` + +### Logic Diagram + +The rough flow of logic that is encapsulated within the `_user_data` internal module can be represented by the following diagram to better highlight the various manners in which user data can be populated. + +

+ User Data +

+ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [cloudinit](#requirement\_cloudinit) | >= 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [cloudinit](#provider\_cloudinit) | >= 2.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [cloudinit_config.linux_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | +| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | +| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | +| [create](#input\_create) | Determines whether to create user-data or not | `bool` | `true` | no | +| [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template | `bool` | `false` | no | +| [is\_eks\_managed\_node\_group](#input\_is\_eks\_managed\_node\_group) | Determines whether the user data is used on nodes in an EKS managed node group. Used to determine if user data will be appended or not | `bool` | `true` | no | +| [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | `string` | `"linux"` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [user\_data](#output\_user\_data) | Base64 encoded user data rendered for the provided inputs | + diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf new file mode 100644 index 0000000000..1d265bdb54 --- /dev/null +++ b/modules/_user_data/main.tf @@ -0,0 +1,78 @@ + +locals { + int_linux_default_user_data = var.create && var.platform == "linux" && (var.enable_bootstrap_user_data || var.user_data_template_path != "") ? base64encode(templatefile( + coalesce(var.user_data_template_path, "${path.module}/../../templates/linux_user_data.tpl"), + { + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami + enable_bootstrap_user_data = var.enable_bootstrap_user_data + # Required to bootstrap node + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + # Optional + cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr != null ? var.cluster_service_ipv4_cidr : "" + bootstrap_extra_args = var.bootstrap_extra_args + pre_bootstrap_user_data = var.pre_bootstrap_user_data + post_bootstrap_user_data = var.post_bootstrap_user_data + } + )) : "" + platform = { + bottlerocket = { + user_data = var.create && var.platform == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? base64encode(templatefile( + coalesce(var.user_data_template_path, "${path.module}/../../templates/bottlerocket_user_data.tpl"), + { + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami + enable_bootstrap_user_data = var.enable_bootstrap_user_data + # Required to bootstrap node + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + # Optional - is appended if using EKS managed node group without custom AMI + # cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr # Not supported yet: https://github.com/bottlerocket-os/bottlerocket/issues/1866 + bootstrap_extra_args = var.bootstrap_extra_args + } + )) : "" + } + linux = { + user_data = try(data.cloudinit_config.linux_eks_managed_node_group[0].rendered, local.int_linux_default_user_data) + + } + windows = { + user_data = var.create && var.platform == "windows" && var.enable_bootstrap_user_data ? base64encode(templatefile( + coalesce(var.user_data_template_path, "${path.module}/../../templates/windows_user_data.tpl"), + { + # Required to bootstrap node + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + # Optional - is appended if using EKS managed node group without custom AMI + # cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr # Not supported yet: https://github.com/awslabs/amazon-eks-ami/issues/805 + bootstrap_extra_args = var.bootstrap_extra_args + pre_bootstrap_user_data = var.pre_bootstrap_user_data + post_bootstrap_user_data = var.post_bootstrap_user_data + } + )) : "" + } + } +} + +# https://github.com/aws/containers-roadmap/issues/596#issuecomment-675097667 +# An important note is that user data must in MIME multi-part archive format, +# as by default, EKS will merge the bootstrapping command required for nodes to join the +# cluster with your user data. If you use a custom AMI in your launch template, +# this merging will NOT happen and you are responsible for nodes joining the cluster. +# See docs for more details -> https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data + +data "cloudinit_config" "linux_eks_managed_node_group" { + count = var.create && var.platform == "linux" && var.is_eks_managed_node_group && !var.enable_bootstrap_user_data && var.pre_bootstrap_user_data != "" && var.user_data_template_path == "" ? 1 : 0 + + base64_encode = true + gzip = false + boundary = "//" + + # Prepend to existing user data suppled by AWS EKS + part { + content_type = "text/x-shellscript" + content = var.pre_bootstrap_user_data + } +} diff --git a/modules/_user_data/outputs.tf b/modules/_user_data/outputs.tf new file mode 100644 index 0000000000..c2a569b05b --- /dev/null +++ b/modules/_user_data/outputs.tf @@ -0,0 +1,4 @@ +output "user_data" { + description = "Base64 encoded user data rendered for the provided inputs" + value = try(local.platform[var.platform].user_data, "") +} diff --git a/modules/_user_data/variables.tf b/modules/_user_data/variables.tf new file mode 100644 index 0000000000..232e1e883e --- /dev/null +++ b/modules/_user_data/variables.tf @@ -0,0 +1,71 @@ +variable "create" { + description = "Determines whether to create user-data or not" + type = bool + default = true +} + +variable "platform" { + description = "Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based" + type = string + default = "linux" +} + +variable "enable_bootstrap_user_data" { + description = "Determines whether the bootstrap configurations are populated within the user data template" + type = bool + default = false +} + +variable "is_eks_managed_node_group" { + description = "Determines whether the user data is used on nodes in an EKS managed node group. Used to determine if user data will be appended or not" + type = bool + default = true +} + +variable "cluster_name" { + description = "Name of the EKS cluster" + type = string + default = "" +} + +variable "cluster_endpoint" { + description = "Endpoint of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_auth_base64" { + description = "Base64 encoded CA of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_service_ipv4_cidr" { + description = "The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" + type = string + default = null +} + +variable "pre_bootstrap_user_data" { + description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + type = string + default = "" +} + +variable "post_bootstrap_user_data" { + description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + type = string + default = "" +} + +variable "bootstrap_extra_args" { + description = "Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" + type = string + default = "" +} + +variable "user_data_template_path" { + description = "Path to a local, custom user data template file to use when rendering user data" + type = string + default = "" +} diff --git a/modules/_user_data/versions.tf b/modules/_user_data/versions.tf new file mode 100644 index 0000000000..e293dc67ce --- /dev/null +++ b/modules/_user_data/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + cloudinit = { + source = "hashicorp/cloudinit" + version = ">= 2.0" + } + } +} diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md new file mode 100644 index 0000000000..43e4a72bc3 --- /dev/null +++ b/modules/eks-managed-node-group/README.md @@ -0,0 +1,175 @@ +# EKS Managed Node Group Module + +Configuration in this directory creates an EKS Managed Node Group along with an IAM role, security group, and launch template + +## Usage + +```hcl +module "eks_managed_node_group" { + source = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group" + + name = "separate-eks-mng" + cluster_name = "my-cluster" + cluster_version = "1.21" + + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + + min_size = 1 + max_size = 10 + desired_size = 1 + + instance_types = ["t3.large"] + capacity_type = "SPOT" + + labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + taints = { + dedicated = { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.64 | +| [cloudinit](#requirement\_cloudinit) | >= 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.64 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [user\_data](#module\_user\_data) | ../_user_data | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_eks_node_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_security_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance. If not supplied, EKS will use its own default image | `string` | `""` | no | +| [ami\_release\_version](#input\_ami\_release\_version) | AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version | `string` | `null` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Valid values are `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `CUSTOM`, `BOTTLEROCKET_ARM_64`, `BOTTLEROCKET_x86_64` | `string` | `null` | no | +| [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `null` | no | +| [capacity\_type](#input\_capacity\_type) | Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT` | `string` | `"ON_DEMAND"` | no | +| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | +| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | +| [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | +| [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | +| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes version. Defaults to EKS Cluster Kubernetes version | `string` | `null` | no | +| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `null` | no | +| [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | +| [create\_security\_group](#input\_create\_security\_group) | Determines whether to create a security group | `bool` | `true` | no | +| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `null` | no | +| [desired\_size](#input\_desired\_size) | Desired number of instances/nodes | `number` | `1` | no | +| [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | +| [disk\_size](#input\_disk\_size) | Disk size in GiB for nodes. Defaults to `20` | `number` | `null` | no | +| [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance(s) will be EBS-optimized | `bool` | `null` | no | +| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `map(string)` | `null` | no | +| [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `null` | no | +| [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template | `bool` | `false` | no | +| [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | +| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `null` | no | +| [force\_update\_version](#input\_force\_update\_version) | Force version update if existing pods are unable to be drained due to a pod disruption budget issue | `bool` | `null` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the node group. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | +| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | +| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `null` | no | +| [instance\_types](#input\_instance\_types) | Set of instance types associated with the EKS Node Group. Defaults to `["t3.medium"]` | `list(string)` | `null` | no | +| [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | +| [key\_name](#input\_key\_name) | The key name that should be used for the instance(s) | `string` | `null` | no | +| [labels](#input\_labels) | Key-value map of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed | `map(string)` | `null` | no | +| [launch\_template\_default\_version](#input\_launch\_template\_default\_version) | Default version of the launch template | `string` | `null` | no | +| [launch\_template\_description](#input\_launch\_template\_description) | Description of the launch template | `string` | `null` | no | +| [launch\_template\_name](#input\_launch\_template\_name) | Launch template name - either to be created (`var.create_launch_template` = `true`) or existing (`var.create_launch_template` = `false`) | `string` | `""` | no | +| [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | +| [launch\_template\_version](#input\_launch\_template\_version) | Launch template version number. The default is `$Default` | `string` | `null` | no | +| [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with | `map(string)` | `null` | no | +| [max\_size](#input\_max\_size) | Maximum number of instances/nodes | `number` | `3` | no | +| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | +| [min\_size](#input\_min\_size) | Minimum number of instances/nodes | `number` | `0` | no | +| [name](#input\_name) | Name of the EKS managed node group | `string` | `""` | no | +| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | +| [placement](#input\_placement) | The placement of the instance | `map(string)` | `null` | no | +| [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket` or `linux` based; `windows` is not supported | `string` | `"linux"` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | +| [remote\_access](#input\_remote\_access) | Configuration block with remote access settings | `map(string)` | `{}` | no | +| [security\_group\_description](#input\_security\_group\_description) | Description for the security group created | `string` | `"EKS managed node group security group"` | no | +| [security\_group\_name](#input\_security\_group\_name) | Name to use on security group created | `string` | `null` | no | +| [security\_group\_rules](#input\_security\_group\_rules) | List of security group rules to add to the security group created | `any` | `{}` | no | +| [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the security group created | `map(string)` | `{}` | no | +| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `string` | `true` | no | +| [subnet\_ids](#input\_subnet\_ids) | Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME` | `list(string)` | `null` | no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | +| [taints](#input\_taints) | The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group | `any` | `{}` | no | +| [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the node group | `map(string)` | `{}` | no | +| [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates | `map(string)` | `{}` | no | +| [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update the launch templates default version on each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | +| [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix | `bool` | `true` | no | +| [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | +| [vpc\_id](#input\_vpc\_id) | ID of the VPC where the security group/nodes will be provisioned | `string` | `null` | no | +| [vpc\_security\_group\_ids](#input\_vpc\_security\_group\_ids) | A list of security group IDs to associate | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | +| [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [launch\_template\_arn](#output\_launch\_template\_arn) | The ARN of the launch template | +| [launch\_template\_id](#output\_launch\_template\_id) | The ID of the launch template | +| [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | +| [node\_group\_arn](#output\_node\_group\_arn) | Amazon Resource Name (ARN) of the EKS Node Group | +| [node\_group\_id](#output\_node\_group\_id) | EKS Cluster name and EKS Node Group name separated by a colon (`:`) | +| [node\_group\_resources](#output\_node\_group\_resources) | List of objects containing information about underlying resources | +| [node\_group\_status](#output\_node\_group\_status) | Status of the EKS Node Group | +| [security\_group\_arn](#output\_security\_group\_arn) | Amazon Resource Name (ARN) of the security group | +| [security\_group\_id](#output\_security\_group\_id) | ID of the security group | + diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf new file mode 100644 index 0000000000..e70e93a239 --- /dev/null +++ b/modules/eks-managed-node-group/main.tf @@ -0,0 +1,435 @@ +data "aws_partition" "current" {} + +################################################################################ +# User Data +################################################################################ + +module "user_data" { + source = "../_user_data" + + create = var.create + platform = var.platform + + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + + cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr + + enable_bootstrap_user_data = var.enable_bootstrap_user_data + pre_bootstrap_user_data = var.pre_bootstrap_user_data + post_bootstrap_user_data = var.post_bootstrap_user_data + bootstrap_extra_args = var.bootstrap_extra_args + user_data_template_path = var.user_data_template_path +} + +################################################################################ +# Launch template +################################################################################ + +locals { + use_custom_launch_template = var.launch_template_name != "" + launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-eks-node-group") +} + +resource "aws_launch_template" "this" { + count = var.create && var.create_launch_template ? 1 : 0 + + name = var.launch_template_use_name_prefix ? null : local.launch_template_name_int + name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name_int}-" : null + description = var.launch_template_description + + ebs_optimized = var.ebs_optimized + image_id = var.ami_id + # # Set on node group instead + # instance_type = var.launch_template_instance_type + key_name = var.key_name + user_data = module.user_data.user_data + + vpc_security_group_ids = compact(concat([try(aws_security_group.this[0].id, "")], var.vpc_security_group_ids)) + + default_version = var.launch_template_default_version + update_default_version = var.update_launch_template_default_version + disable_api_termination = var.disable_api_termination + # Set on EKS managed node group, will fail if set here + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics + # instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior + kernel_id = var.kernel_id + ram_disk_id = var.ram_disk_id + + dynamic "block_device_mappings" { + for_each = var.block_device_mappings + content { + device_name = block_device_mappings.value.device_name + no_device = lookup(block_device_mappings.value, "no_device", null) + virtual_name = lookup(block_device_mappings.value, "virtual_name", null) + + dynamic "ebs" { + for_each = flatten([lookup(block_device_mappings.value, "ebs", [])]) + content { + delete_on_termination = lookup(ebs.value, "delete_on_termination", null) + encrypted = lookup(ebs.value, "encrypted", null) + kms_key_id = lookup(ebs.value, "kms_key_id", null) + iops = lookup(ebs.value, "iops", null) + throughput = lookup(ebs.value, "throughput", null) + snapshot_id = lookup(ebs.value, "snapshot_id", null) + volume_size = lookup(ebs.value, "volume_size", null) + volume_type = lookup(ebs.value, "volume_type", null) + } + } + } + } + + dynamic "capacity_reservation_specification" { + for_each = var.capacity_reservation_specification != null ? [var.capacity_reservation_specification] : [] + content { + capacity_reservation_preference = lookup(capacity_reservation_specification.value, "capacity_reservation_preference", null) + + dynamic "capacity_reservation_target" { + for_each = lookup(capacity_reservation_specification.value, "capacity_reservation_target", []) + content { + capacity_reservation_id = lookup(capacity_reservation_target.value, "capacity_reservation_id", null) + } + } + } + } + + dynamic "cpu_options" { + for_each = var.cpu_options != null ? [var.cpu_options] : [] + content { + core_count = cpu_options.value.core_count + threads_per_core = cpu_options.value.threads_per_core + } + } + + dynamic "credit_specification" { + for_each = var.credit_specification != null ? [var.credit_specification] : [] + content { + cpu_credits = credit_specification.value.cpu_credits + } + } + + dynamic "elastic_gpu_specifications" { + for_each = var.elastic_gpu_specifications != null ? [var.elastic_gpu_specifications] : [] + content { + type = elastic_gpu_specifications.value.type + } + } + + dynamic "elastic_inference_accelerator" { + for_each = var.elastic_inference_accelerator != null ? [var.elastic_inference_accelerator] : [] + content { + type = elastic_inference_accelerator.value.type + } + } + + dynamic "enclave_options" { + for_each = var.enclave_options != null ? [var.enclave_options] : [] + content { + enabled = enclave_options.value.enabled + } + } + + # Set on EKS managed node group, will fail if set here + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics + # dynamic "hibernation_options" { + # for_each = var.hibernation_options != null ? [var.hibernation_options] : [] + # content { + # configured = hibernation_options.value.configured + # } + # } + + # Set on EKS managed node group, will fail if set here + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics + # dynamic "iam_instance_profile" { + # for_each = [var.iam_instance_profile] + # content { + # name = lookup(var.iam_instance_profile, "name", null) + # arn = lookup(var.iam_instance_profile, "arn", null) + # } + # } + + dynamic "instance_market_options" { + for_each = var.instance_market_options != null ? [var.instance_market_options] : [] + content { + market_type = instance_market_options.value.market_type + + dynamic "spot_options" { + for_each = lookup(instance_market_options.value, "spot_options", null) != null ? [instance_market_options.value.spot_options] : [] + content { + block_duration_minutes = spot_options.value.block_duration_minutes + instance_interruption_behavior = lookup(spot_options.value, "instance_interruption_behavior", null) + max_price = lookup(spot_options.value, "max_price", null) + spot_instance_type = lookup(spot_options.value, "spot_instance_type", null) + valid_until = lookup(spot_options.value, "valid_until", null) + } + } + } + } + + dynamic "license_specification" { + for_each = var.license_specifications != null ? [var.license_specifications] : [] + content { + license_configuration_arn = license_specifications.value.license_configuration_arn + } + } + + dynamic "metadata_options" { + for_each = var.metadata_options != null ? [var.metadata_options] : [] + content { + http_endpoint = lookup(metadata_options.value, "http_endpoint", null) + http_tokens = lookup(metadata_options.value, "http_tokens", null) + http_put_response_hop_limit = lookup(metadata_options.value, "http_put_response_hop_limit", null) + http_protocol_ipv6 = lookup(metadata_options.value, "http_protocol_ipv6", null) + } + } + + dynamic "monitoring" { + for_each = var.enable_monitoring != null ? [1] : [] + content { + enabled = var.enable_monitoring + } + } + + dynamic "network_interfaces" { + for_each = var.network_interfaces + content { + associate_carrier_ip_address = lookup(network_interfaces.value, "associate_carrier_ip_address", null) + associate_public_ip_address = lookup(network_interfaces.value, "associate_public_ip_address", null) + delete_on_termination = lookup(network_interfaces.value, "delete_on_termination", null) + description = lookup(network_interfaces.value, "description", null) + device_index = lookup(network_interfaces.value, "device_index", null) + ipv4_addresses = lookup(network_interfaces.value, "ipv4_addresses", null) != null ? network_interfaces.value.ipv4_addresses : [] + ipv4_address_count = lookup(network_interfaces.value, "ipv4_address_count", null) + ipv6_addresses = lookup(network_interfaces.value, "ipv6_addresses", null) != null ? network_interfaces.value.ipv6_addresses : [] + ipv6_address_count = lookup(network_interfaces.value, "ipv6_address_count", null) + network_interface_id = lookup(network_interfaces.value, "network_interface_id", null) + private_ip_address = lookup(network_interfaces.value, "private_ip_address", null) + security_groups = lookup(network_interfaces.value, "security_groups", null) != null ? network_interfaces.value.security_groups : [] + # Set on EKS managed node group, will fail if set here + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics + # subnet_id = lookup(network_interfaces.value, "subnet_id", null) + } + } + + dynamic "placement" { + for_each = var.placement != null ? [var.placement] : [] + content { + affinity = lookup(placement.value, "affinity", null) + availability_zone = lookup(placement.value, "availability_zone", null) + group_name = lookup(placement.value, "group_name", null) + host_id = lookup(placement.value, "host_id", null) + spread_domain = lookup(placement.value, "spread_domain", null) + tenancy = lookup(placement.value, "tenancy", null) + partition_number = lookup(placement.value, "partition_number", null) + } + } + + dynamic "tag_specifications" { + for_each = toset(["instance", "volume", "network-interface"]) + content { + resource_type = tag_specifications.key + tags = merge(var.tags, { Name = var.name }) + } + } + + lifecycle { + create_before_destroy = true + } + + # Prevent premature access of security group roles and policies by pods that + # require permissions on create/destroy that depend on nodes + depends_on = [ + aws_security_group_rule.this, + aws_iam_role_policy_attachment.this, + ] + + tags = var.tags +} + +################################################################################ +# Node Group +################################################################################ + +locals { + launch_template_name = try(aws_launch_template.this[0].name, var.launch_template_name, null) + # Change order to allow users to set version priority before using defaults + launch_template_version = coalesce(var.launch_template_version, try(aws_launch_template.this[0].default_version, "$Default")) +} + +resource "aws_eks_node_group" "this" { + count = var.create ? 1 : 0 + + # Required + cluster_name = var.cluster_name + node_role_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn + subnet_ids = var.subnet_ids + + scaling_config { + min_size = var.min_size + max_size = var.max_size + desired_size = var.desired_size + } + + # Optional + node_group_name = var.use_name_prefix ? null : var.name + node_group_name_prefix = var.use_name_prefix ? "${var.name}-" : null + + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami + ami_type = var.ami_id != "" ? null : var.ami_type + release_version = var.ami_id != "" ? null : var.ami_release_version + version = var.ami_id != "" ? null : var.cluster_version + + capacity_type = var.capacity_type + disk_size = local.use_custom_launch_template ? null : var.disk_size # if using LT, set disk size on LT or else it will error here + force_update_version = var.force_update_version + instance_types = var.instance_types + labels = var.labels + + dynamic "launch_template" { + for_each = local.use_custom_launch_template ? [1] : [] + content { + name = local.launch_template_name + version = local.launch_template_version + } + } + + dynamic "remote_access" { + for_each = var.remote_access + content { + ec2_ssh_key = lookup(remote_access.value, "ec2_ssh_key", null) + source_security_group_ids = lookup(remote_access.value, "source_security_group_ids", []) + } + } + + dynamic "taint" { + for_each = var.taints + content { + key = taint.value.key + value = lookup(taint.value, "value") + effect = taint.value.effect + } + } + + dynamic "update_config" { + for_each = length(var.update_config) > 0 ? [var.update_config] : [] + content { + max_unavailable_percentage = try(update_config.value.max_unavailable_percentage, null) + max_unavailable = try(update_config.value.max_unavailable, null) + } + } + + timeouts { + create = lookup(var.timeouts, "create", null) + update = lookup(var.timeouts, "update", null) + delete = lookup(var.timeouts, "delete", null) + } + + lifecycle { + create_before_destroy = true + ignore_changes = [ + scaling_config[0].desired_size, + ] + } + + tags = merge( + var.tags, + { Name = var.name } + ) +} + +################################################################################ +# Security Group +################################################################################ + +locals { + security_group_name = coalesce(var.security_group_name, "${var.name}-eks-node-group") + create_security_group = var.create && var.create_security_group +} + +resource "aws_security_group" "this" { + count = local.create_security_group ? 1 : 0 + + name = var.security_group_use_name_prefix ? null : local.security_group_name + name_prefix = var.security_group_use_name_prefix ? "${local.security_group_name}-" : null + description = var.security_group_description + vpc_id = var.vpc_id + + tags = merge( + var.tags, + { "Name" = local.security_group_name }, + var.security_group_tags + ) +} + +resource "aws_security_group_rule" "this" { + for_each = { for k, v in var.security_group_rules : k => v if local.create_security_group } + + # Required + security_group_id = aws_security_group.this[0].id + protocol = each.value.protocol + from_port = each.value.from_port + to_port = each.value.to_port + type = each.value.type + + # Optional + description = try(each.value.description, null) + cidr_blocks = try(each.value.cidr_blocks, null) + ipv6_cidr_blocks = try(each.value.ipv6_cidr_blocks, null) + prefix_list_ids = try(each.value.prefix_list_ids, []) + self = try(each.value.self, null) + source_security_group_id = try( + each.value.source_security_group_id, + try(each.value.source_cluster_security_group, false) ? var.cluster_security_group_id : null + ) +} + +################################################################################ +# IAM Role +################################################################################ + +locals { + iam_role_name = coalesce(var.iam_role_name, "${var.name}-eks-node-group") + policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" +} + +data "aws_iam_policy_document" "assume_role_policy" { + count = var.create && var.create_iam_role ? 1 : 0 + + statement { + sid = "EKSNodeAssumeRole" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.${data.aws_partition.current.dns_suffix}"] + } + } +} + +resource "aws_iam_role" "this" { + count = var.create && var.create_iam_role ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + assume_role_policy = data.aws_iam_policy_document.assume_role_policy[0].json + permissions_boundary = var.iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +# Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group +resource "aws_iam_role_policy_attachment" "this" { + for_each = var.create && var.create_iam_role ? toset(compact(distinct(concat([ + "${local.policy_arn_prefix}/AmazonEKSWorkerNodePolicy", + "${local.policy_arn_prefix}/AmazonEC2ContainerRegistryReadOnly", + "${local.policy_arn_prefix}/AmazonEKS_CNI_Policy", + ], var.iam_role_additional_policies)))) : toset([]) + + policy_arn = each.value + role = aws_iam_role.this[0].name +} diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf new file mode 100644 index 0000000000..476abc8420 --- /dev/null +++ b/modules/eks-managed-node-group/outputs.tf @@ -0,0 +1,75 @@ +################################################################################ +# Launch template +################################################################################ + +output "launch_template_id" { + description = "The ID of the launch template" + value = try(aws_launch_template.this[0].id, "") +} + +output "launch_template_arn" { + description = "The ARN of the launch template" + value = try(aws_launch_template.this[0].arn, "") +} + +output "launch_template_latest_version" { + description = "The latest version of the launch template" + value = try(aws_launch_template.this[0].latest_version, "") +} + +################################################################################ +# Node Group +################################################################################ + +output "node_group_arn" { + description = "Amazon Resource Name (ARN) of the EKS Node Group" + value = try(aws_eks_node_group.this[0].arn, "") +} + +output "node_group_id" { + description = "EKS Cluster name and EKS Node Group name separated by a colon (`:`)" + value = try(aws_eks_node_group.this[0].id, "") +} + +output "node_group_resources" { + description = "List of objects containing information about underlying resources" + value = try(aws_eks_node_group.this[0].resources, "") +} + +output "node_group_status" { + description = "Status of the EKS Node Group" + value = try(aws_eks_node_group.this[0].arn, "") +} + +################################################################################ +# Security Group +################################################################################ + +output "security_group_arn" { + description = "Amazon Resource Name (ARN) of the security group" + value = try(aws_security_group.this[0].arn, "") +} + +output "security_group_id" { + description = "ID of the security group" + value = try(aws_security_group.this[0].id, "") +} + +################################################################################ +# IAM Role +################################################################################ + +output "iam_role_name" { + description = "The name of the IAM role" + value = try(aws_iam_role.this[0].name, "") +} + +output "iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = try(aws_iam_role.this[0].arn, "") +} + +output "iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, "") +} diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf new file mode 100644 index 0000000000..38d126f269 --- /dev/null +++ b/modules/eks-managed-node-group/variables.tf @@ -0,0 +1,467 @@ +variable "create" { + description = "Determines whether to create EKS managed node group or not" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +variable "platform" { + description = "Identifies if the OS platform is `bottlerocket` or `linux` based; `windows` is not supported" + type = string + default = "linux" +} + +################################################################################ +# User Data +################################################################################ + +variable "enable_bootstrap_user_data" { + description = "Determines whether the bootstrap configurations are populated within the user data template" + type = bool + default = false +} + +variable "cluster_name" { + description = "Name of associated EKS cluster" + type = string + default = null +} + +variable "cluster_endpoint" { + description = "Endpoint of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_auth_base64" { + description = "Base64 encoded CA of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_service_ipv4_cidr" { + description = "The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" + type = string + default = null +} + +variable "pre_bootstrap_user_data" { + description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + type = string + default = "" +} + +variable "post_bootstrap_user_data" { + description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + type = string + default = "" +} + +variable "bootstrap_extra_args" { + description = "Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" + type = string + default = "" +} + +variable "user_data_template_path" { + description = "Path to a local, custom user data template file to use when rendering user data" + type = string + default = "" +} + +################################################################################ +# Launch template +################################################################################ + +variable "create_launch_template" { + description = "Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template" + type = bool + default = true +} + +variable "launch_template_name" { + description = "Launch template name - either to be created (`var.create_launch_template` = `true`) or existing (`var.create_launch_template` = `false`)" + type = string + default = "" +} + +variable "launch_template_use_name_prefix" { + description = "Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix" + type = bool + default = true +} + +variable "launch_template_description" { + description = "Description of the launch template" + type = string + default = null +} + +variable "ebs_optimized" { + description = "If true, the launched EC2 instance(s) will be EBS-optimized" + type = bool + default = null +} + +variable "ami_id" { + description = "The AMI from which to launch the instance. If not supplied, EKS will use its own default image" + type = string + default = "" +} + +variable "key_name" { + description = "The key name that should be used for the instance(s)" + type = string + default = null +} + +variable "vpc_security_group_ids" { + description = "A list of security group IDs to associate" + type = list(string) + default = [] +} + +variable "launch_template_default_version" { + description = "Default version of the launch template" + type = string + default = null +} + +variable "update_launch_template_default_version" { + description = "Whether to update the launch templates default version on each update. Conflicts with `launch_template_default_version`" + type = bool + default = true +} + +variable "disable_api_termination" { + description = "If true, enables EC2 instance termination protection" + type = bool + default = null +} + +variable "kernel_id" { + description = "The kernel ID" + type = string + default = null +} + +variable "ram_disk_id" { + description = "The ID of the ram disk" + type = string + default = null +} + +variable "block_device_mappings" { + description = "Specify volumes to attach to the instance besides the volumes specified by the AMI" + type = any + default = {} +} + +variable "capacity_reservation_specification" { + description = "Targeting for EC2 capacity reservations" + type = any + default = null +} + +variable "cpu_options" { + description = "The CPU options for the instance" + type = map(string) + default = null +} + +variable "credit_specification" { + description = "Customize the credit specification of the instance" + type = map(string) + default = null +} + +variable "elastic_gpu_specifications" { + description = "The elastic GPU to attach to the instance" + type = map(string) + default = null +} + +variable "elastic_inference_accelerator" { + description = "Configuration block containing an Elastic Inference Accelerator to attach to the instance" + type = map(string) + default = null +} + +variable "enclave_options" { + description = "Enable Nitro Enclaves on launched instances" + type = map(string) + default = null +} + +variable "instance_market_options" { + description = "The market (purchasing) option for the instance" + type = any + default = null +} + +variable "license_specifications" { + description = "A list of license specifications to associate with" + type = map(string) + default = null +} + +variable "metadata_options" { + description = "Customize the metadata options for the instance" + type = map(string) + default = { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 2 + } +} + +variable "enable_monitoring" { + description = "Enables/disables detailed monitoring" + type = bool + default = true +} + +variable "network_interfaces" { + description = "Customize network interfaces to be attached at instance boot time" + type = list(any) + default = [] +} + +variable "placement" { + description = "The placement of the instance" + type = map(string) + default = null +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +variable "subnet_ids" { + description = "Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME`" + type = list(string) + default = null +} + +variable "min_size" { + description = "Minimum number of instances/nodes" + type = number + default = 0 +} + +variable "max_size" { + description = "Maximum number of instances/nodes" + type = number + default = 3 +} + +variable "desired_size" { + description = "Desired number of instances/nodes" + type = number + default = 1 +} + +variable "name" { + description = "Name of the EKS managed node group" + type = string + default = "" +} + +variable "use_name_prefix" { + description = "Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix" + type = bool + default = true +} + +variable "ami_type" { + description = "Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Valid values are `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `CUSTOM`, `BOTTLEROCKET_ARM_64`, `BOTTLEROCKET_x86_64`" + type = string + default = null +} + +variable "ami_release_version" { + description = "AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version" + type = string + default = null +} + +variable "capacity_type" { + description = "Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT`" + type = string + default = "ON_DEMAND" +} + +variable "disk_size" { + description = "Disk size in GiB for nodes. Defaults to `20`" + type = number + default = null +} + +variable "force_update_version" { + description = "Force version update if existing pods are unable to be drained due to a pod disruption budget issue" + type = bool + default = null +} + +variable "instance_types" { + description = "Set of instance types associated with the EKS Node Group. Defaults to `[\"t3.medium\"]`" + type = list(string) + default = null +} + +variable "labels" { + description = "Key-value map of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed" + type = map(string) + default = null +} + +variable "cluster_version" { + description = "Kubernetes version. Defaults to EKS Cluster Kubernetes version" + type = string + default = null +} + +variable "launch_template_version" { + description = "Launch template version number. The default is `$Default`" + type = string + default = null +} + +variable "remote_access" { + description = "Configuration block with remote access settings" + type = map(string) + default = {} +} + +variable "taints" { + description = "The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group" + type = any + default = {} +} + +variable "update_config" { + description = "Configuration block of settings for max unavailable resources during node group updates" + type = map(string) + default = {} +} + +variable "timeouts" { + description = "Create, update, and delete timeout configurations for the node group" + type = map(string) + default = {} +} + +################################################################################ +# Security Group +################################################################################ + +variable "create_security_group" { + description = "Determines whether to create a security group" + type = bool + default = true +} + +variable "security_group_name" { + description = "Name to use on security group created" + type = string + default = null +} + +variable "security_group_use_name_prefix" { + description = "Determines whether the security group name (`security_group_name`) is used as a prefix" + type = string + default = true +} + +variable "security_group_description" { + description = "Description for the security group created" + type = string + default = "EKS managed node group security group" +} + +variable "vpc_id" { + description = "ID of the VPC where the security group/nodes will be provisioned" + type = string + default = null +} + +variable "security_group_rules" { + description = "List of security group rules to add to the security group created" + type = any + default = {} +} + +variable "cluster_security_group_id" { + description = "Cluster control plane security group ID" + type = string + default = null +} + +variable "security_group_tags" { + description = "A map of additional tags to add to the security group created" + type = map(string) + default = {} +} + +################################################################################ +# IAM Role +################################################################################ + +variable "create_iam_role" { + description = "Determines whether an IAM role is created or to use an existing IAM role" + type = bool + default = true +} + +variable "iam_role_arn" { + description = "Existing IAM role ARN for the node group. Required if `create_iam_role` is set to `false`" + type = string + default = null +} + +variable "iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" + type = string + default = true +} + +variable "iam_role_path" { + description = "IAM role path" + type = string + default = null +} + +variable "iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = list(string) + default = [] +} + +variable "iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} diff --git a/modules/node_groups/versions.tf b/modules/eks-managed-node-group/versions.tf similarity index 89% rename from modules/node_groups/versions.tf rename to modules/eks-managed-node-group/versions.tf index f4d0036a86..48492037e2 100644 --- a/modules/node_groups/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.56" + version = ">= 3.64" } cloudinit = { source = "hashicorp/cloudinit" diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md new file mode 100644 index 0000000000..8a51f317a8 --- /dev/null +++ b/modules/fargate-profile/README.md @@ -0,0 +1,85 @@ +# EKS Fargate Profile Module + +Configuration in this directory creates a Fargate EKS Profile + +## Usage + +```hcl +module "fargate_profile" { + source = "terraform-aws-modules/eks/aws//modules/fargate-profile" + + name = "separate-fargate-profile" + cluster_name = "my-cluster" + + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + selectors = [{ + namespace = "kube-system" + }] + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.64 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.64 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_eks_fargate_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_fargate_profile) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `null` | no | +| [create](#input\_create) | Determines whether to create Fargate profile or not | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the Fargate profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `""` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | +| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | +| [name](#input\_name) | Name of the EKS Fargate Profile | `string` | `""` | no | +| [selectors](#input\_selectors) | Configuration block(s) for selecting Kubernetes Pods to execute with this Fargate Profile | `any` | `[]` | no | +| [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs for the EKS Fargate Profile | `list(string)` | `[]` | no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | +| [timeouts](#input\_timeouts) | Create and delete timeout configurations for the Fargate Profile | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [fargate\_profile\_arn](#output\_fargate\_profile\_arn) | Amazon Resource Name (ARN) of the EKS Fargate Profile | +| [fargate\_profile\_id](#output\_fargate\_profile\_id) | EKS Cluster name and EKS Fargate Profile name separated by a colon (`:`) | +| [fargate\_profile\_status](#output\_fargate\_profile\_status) | Status of the EKS Fargate Profile | +| [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | +| [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | + diff --git a/modules/fargate-profile/main.tf b/modules/fargate-profile/main.tf new file mode 100644 index 0000000000..b37fc45116 --- /dev/null +++ b/modules/fargate-profile/main.tf @@ -0,0 +1,80 @@ +data "aws_partition" "current" {} + +locals { + iam_role_name = coalesce(var.iam_role_name, var.name, "fargate-profile") + policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" +} + +################################################################################ +# IAM Role +################################################################################ + +data "aws_iam_policy_document" "assume_role_policy" { + count = var.create && var.create_iam_role ? 1 : 0 + + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["eks-fargate-pods.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "this" { + count = var.create && var.create_iam_role ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + assume_role_policy = data.aws_iam_policy_document.assume_role_policy[0].json + permissions_boundary = var.iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +resource "aws_iam_role_policy_attachment" "this" { + for_each = var.create && var.create_iam_role ? toset(compact(distinct(concat([ + "${local.policy_arn_prefix}/AmazonEKSFargatePodExecutionRolePolicy", + ], var.iam_role_additional_policies)))) : toset([]) + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +################################################################################ +# Fargate Profile +################################################################################ + +resource "aws_eks_fargate_profile" "this" { + count = var.create ? 1 : 0 + + cluster_name = var.cluster_name + fargate_profile_name = var.name + pod_execution_role_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn + subnet_ids = var.subnet_ids + + dynamic "selector" { + for_each = var.selectors + + content { + namespace = selector.value.namespace + labels = lookup(selector.value, "labels", {}) + } + } + + dynamic "timeouts" { + for_each = [var.timeouts] + content { + create = lookup(var.timeouts, "create", null) + delete = lookup(var.timeouts, "delete", null) + } + } + + tags = var.tags +} diff --git a/modules/fargate-profile/outputs.tf b/modules/fargate-profile/outputs.tf new file mode 100644 index 0000000000..bb6e023248 --- /dev/null +++ b/modules/fargate-profile/outputs.tf @@ -0,0 +1,37 @@ +################################################################################ +# IAM Role +################################################################################ + +output "iam_role_name" { + description = "The name of the IAM role" + value = try(aws_iam_role.this[0].name, "") +} + +output "iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = try(aws_iam_role.this[0].arn, "") +} + +output "iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, "") +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profile_arn" { + description = "Amazon Resource Name (ARN) of the EKS Fargate Profile" + value = try(aws_eks_fargate_profile.this[0].arn, "") +} + +output "fargate_profile_id" { + description = "EKS Cluster name and EKS Fargate Profile name separated by a colon (`:`)" + value = try(aws_eks_fargate_profile.this[0].id, "") +} + +output "fargate_profile_status" { + description = "Status of the EKS Fargate Profile" + value = try(aws_eks_fargate_profile.this[0].status, "") +} diff --git a/modules/fargate-profile/variables.tf b/modules/fargate-profile/variables.tf new file mode 100644 index 0000000000..afad4809a6 --- /dev/null +++ b/modules/fargate-profile/variables.tf @@ -0,0 +1,103 @@ +variable "create" { + description = "Determines whether to create Fargate profile or not" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +################################################################################ +# IAM Role +################################################################################ + +variable "create_iam_role" { + description = "Determines whether an IAM role is created or to use an existing IAM role" + type = bool + default = true +} + +variable "iam_role_arn" { + description = "Existing IAM role ARN for the Fargate profile. Required if `create_iam_role` is set to `false`" + type = string + default = null +} + +variable "iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = "" +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" + type = string + default = true +} + +variable "iam_role_path" { + description = "IAM role path" + type = string + default = null +} + +variable "iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = list(string) + default = [] +} + +variable "iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} + +################################################################################ +# Fargate Profile +################################################################################ + +variable "cluster_name" { + description = "Name of the EKS cluster" + type = string + default = null +} + +variable "name" { + description = "Name of the EKS Fargate Profile" + type = string + default = "" +} + +variable "subnet_ids" { + description = "A list of subnet IDs for the EKS Fargate Profile" + type = list(string) + default = [] +} + +variable "selectors" { + description = "Configuration block(s) for selecting Kubernetes Pods to execute with this Fargate Profile" + type = any + default = [] +} + +variable "timeouts" { + description = "Create and delete timeout configurations for the Fargate Profile" + type = map(string) + default = {} +} diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf new file mode 100644 index 0000000000..bfce6ae345 --- /dev/null +++ b/modules/fargate-profile/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.64" + } + } +} diff --git a/modules/fargate/README.md b/modules/fargate/README.md deleted file mode 100644 index 7a20be175b..0000000000 --- a/modules/fargate/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# EKS `fargate` submodule - -Helper submodule to create and manage resources related to `aws_eks_fargate_profile`. - -## `fargate_profile` keys - -`fargate_profile` is a map of maps. Key of first level will be used as unique value for `for_each` resources and in the `aws_eks_fargate_profile` name. Inner map can take the below values. - -## Example - -See example code in `examples/fargate`. - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| name | Fargate profile name | `string` | Auto generated in the following format `[cluster_name]-fargate-[fargate_profile_map_key]`| no | -| selectors | A list of Kubernetes selectors. See examples/fargate/main.tf for example format. |
list(map({
namespace = string
labels = map(string)
}))
| `[]` | no | -| subnets | List of subnet IDs. Will replace the root module subnets. | `list(string)` | `var.subnets` | no | -| timeouts | A map of timeouts for create/delete operations. | `map(string)` | Provider default behavior | no | -| tags | Key-value map of resource tags. Will be merged with root module tags. | `map(string)` | `var.tags` | no | - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.56 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [aws_eks_fargate_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_fargate_profile) | resource | -| [aws_iam_role.eks_fargate_pod](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role_policy_attachment.eks_fargate_pod](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_policy_document.eks_fargate_pod_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_role.custom_fargate_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | -| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster. | `string` | `""` | no | -| [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | -| [create\_fargate\_pod\_execution\_role](#input\_create\_fargate\_pod\_execution\_role) | Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created. | `bool` | `true` | no | -| [fargate\_pod\_execution\_role\_name](#input\_fargate\_pod\_execution\_role\_name) | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | -| [fargate\_profiles](#input\_fargate\_profiles) | Fargate profiles to create. See `fargate_profile` keys section in README.md for more details | `any` | `{}` | no | -| [iam\_path](#input\_iam\_path) | IAM roles will be created on this path. | `string` | `"/"` | no | -| [permissions\_boundary](#input\_permissions\_boundary) | If provided, all IAM roles will be created with this permissions boundary attached. | `string` | `null` | no | -| [subnets](#input\_subnets) | A list of subnets for the EKS Fargate profiles. | `list(string)` | `[]` | no | -| [tags](#input\_tags) | A map of tags to add to all resources. | `map(string)` | `{}` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [aws\_auth\_roles](#output\_aws\_auth\_roles) | Roles for use in aws-auth ConfigMap | -| [fargate\_profile\_arns](#output\_fargate\_profile\_arns) | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | -| [fargate\_profile\_ids](#output\_fargate\_profile\_ids) | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | -| [iam\_role\_arn](#output\_iam\_role\_arn) | IAM role ARN for EKS Fargate pods | -| [iam\_role\_name](#output\_iam\_role\_name) | IAM role name for EKS Fargate pods | - diff --git a/modules/fargate/main.tf b/modules/fargate/main.tf deleted file mode 100644 index a4e4b0d80f..0000000000 --- a/modules/fargate/main.tf +++ /dev/null @@ -1,72 +0,0 @@ -locals { - create_eks = var.create_eks && length(var.fargate_profiles) > 0 - - pod_execution_role_arn = coalescelist(aws_iam_role.eks_fargate_pod.*.arn, data.aws_iam_role.custom_fargate_iam_role.*.arn, [""])[0] - pod_execution_role_name = coalescelist(aws_iam_role.eks_fargate_pod.*.name, data.aws_iam_role.custom_fargate_iam_role.*.name, [""])[0] - - fargate_profiles = { for k, v in var.fargate_profiles : k => v if var.create_eks } -} - -data "aws_partition" "current" {} - -data "aws_iam_policy_document" "eks_fargate_pod_assume_role" { - count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 - - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["eks-fargate-pods.amazonaws.com"] - } - } -} - -data "aws_iam_role" "custom_fargate_iam_role" { - count = local.create_eks && !var.create_fargate_pod_execution_role ? 1 : 0 - - name = var.fargate_pod_execution_role_name -} - -resource "aws_iam_role" "eks_fargate_pod" { - count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 - - name_prefix = format("%s-fargate", substr(var.cluster_name, 0, 24)) - assume_role_policy = data.aws_iam_policy_document.eks_fargate_pod_assume_role[0].json - permissions_boundary = var.permissions_boundary - tags = var.tags - path = var.iam_path -} - -resource "aws_iam_role_policy_attachment" "eks_fargate_pod" { - count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 - - policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy" - role = aws_iam_role.eks_fargate_pod[0].name -} - -resource "aws_eks_fargate_profile" "this" { - for_each = local.fargate_profiles - - cluster_name = var.cluster_name - fargate_profile_name = lookup(each.value, "name", format("%s-fargate-%s", var.cluster_name, replace(each.key, "_", "-"))) - pod_execution_role_arn = local.pod_execution_role_arn - subnet_ids = lookup(each.value, "subnets", var.subnets) - - dynamic "selector" { - for_each = each.value.selectors - - content { - namespace = selector.value["namespace"] - labels = lookup(selector.value, "labels", {}) - } - } - - timeouts { - create = try(each.value["timeouts"].create, null) - delete = try(each.value["timeouts"].delete, null) - } - - tags = merge(var.tags, lookup(each.value, "tags", {})) -} diff --git a/modules/fargate/outputs.tf b/modules/fargate/outputs.tf deleted file mode 100644 index 126ba6e385..0000000000 --- a/modules/fargate/outputs.tf +++ /dev/null @@ -1,29 +0,0 @@ -output "fargate_profile_ids" { - description = "EKS Cluster name and EKS Fargate Profile names separated by a colon (:)." - value = [for f in aws_eks_fargate_profile.this : f.id] -} - -output "fargate_profile_arns" { - description = "Amazon Resource Name (ARN) of the EKS Fargate Profiles." - value = [for f in aws_eks_fargate_profile.this : f.arn] -} - -output "iam_role_name" { - description = "IAM role name for EKS Fargate pods" - value = local.pod_execution_role_name -} - -output "iam_role_arn" { - description = "IAM role ARN for EKS Fargate pods" - value = local.pod_execution_role_arn -} - -output "aws_auth_roles" { - description = "Roles for use in aws-auth ConfigMap" - value = [ - for i in range(1) : { - worker_role_arn = local.pod_execution_role_arn - platform = "fargate" - } if local.create_eks - ] -} diff --git a/modules/fargate/variables.tf b/modules/fargate/variables.tf deleted file mode 100644 index 39e2cc68b3..0000000000 --- a/modules/fargate/variables.tf +++ /dev/null @@ -1,53 +0,0 @@ -variable "create_eks" { - description = "Controls if EKS resources should be created (it affects almost all resources)" - type = bool - default = true -} - -variable "create_fargate_pod_execution_role" { - description = "Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created." - type = bool - default = true -} - -variable "cluster_name" { - description = "Name of the EKS cluster." - type = string - default = "" -} - -variable "iam_path" { - description = "IAM roles will be created on this path." - type = string - default = "/" -} - -variable "fargate_pod_execution_role_name" { - description = "The IAM Role that provides permissions for the EKS Fargate Profile." - type = string - default = null -} - -variable "fargate_profiles" { - description = "Fargate profiles to create. See `fargate_profile` keys section in README.md for more details" - type = any - default = {} -} - -variable "permissions_boundary" { - description = "If provided, all IAM roles will be created with this permissions boundary attached." - type = string - default = null -} - -variable "subnets" { - description = "A list of subnets for the EKS Fargate profiles." - type = list(string) - default = [] -} - -variable "tags" { - description = "A map of tags to add to all resources." - type = map(string) - default = {} -} diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md deleted file mode 100644 index f91ce04ae0..0000000000 --- a/modules/node_groups/README.md +++ /dev/null @@ -1,113 +0,0 @@ -# EKS `node_groups` submodule - -Helper submodule to create and manage resources related to `eks_node_groups`. - -## Node Groups' IAM Role - -The role ARN specified in `var.default_iam_role_arn` will be used by default. In a simple configuration this will be the worker role created by the parent module. - -`iam_role_arn` must be specified in either `var.node_groups_defaults` or `var.node_groups` if the default parent IAM role is not being created for whatever reason, for example if `manage_worker_iam_resources` is set to false in the parent. - -## `node_groups` and `node_groups_defaults` keys -`node_groups_defaults` is a map that can take the below keys. Values will be used if not specified in individual node groups. - -`node_groups` is a map of maps. Key of first level will be used as unique value for `for_each` resources and in the `aws_eks_node_group` name. Inner map can take the below values. - -| Name | Description | Type | If unset | -|------|-------------|:----:|:-----:| -| additional\_tags | Additional tags to apply to node group | map(string) | Only `var.tags` applied | -| ami\_release\_version | AMI version of workers | string | Provider default behavior | -| ami\_type | AMI Type. See Terraform or AWS docs | string | Provider default behavior | -| ami\_id | ID of custom AMI. If you use a custom AMI, you need to set `ami_is_eks_optimized` | string | Provider default behavior | -| ami\_is\_eks\_optimized | If the custom AMI is an EKS optimised image, ignored if `ami_id` is not set. If this is `true` then `bootstrap.sh` is called automatically (max pod logic needs to be manually set), if this is `false` you need to provide all the node configuration in `pre_userdata` | bool | `true` | -| capacity\_type | Type of instance capacity to provision. Options are `ON_DEMAND` and `SPOT` | string | Provider default behavior | -| create_launch_template | Create and use a default launch template | bool | `false` | -| desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` | -| disk\_encrypted | Whether the root disk will be encrypyted. Requires `create_launch_template` to be `true` and `disk_kms_key_id` to be set | bool | false | -| disk\_kms\_key\_id | KMS Key used to encrypt the root disk. Requires both `create_launch_template` and `disk_encrypted` to be `true` | string | "" | -| disk\_size | Workers' disk size | number | Provider default behavior | -| disk\_type | Workers' disk type. Require `create_launch_template` to be `true`| string | Provider default behavior | -| disk\_throughput | Workers' disk throughput. Require `create_launch_template` to be `true` and `disk_type` to be `gp3`| number | Provider default behavior | -| disk\_iops | Workers' disk IOPS. Require `create_launch_template` to be `true` and `disk_type` to be `gp3`| number | Provider default behavior | -| ebs\_optimized | Enables/disables EBS optimization. Require `create_launch_template` to be `true` | bool | `true` if defined `instance\_types` are not present in `var.ebs\_optimized\_not\_supported` | -| enable_monitoring | Enables/disables detailed monitoring. Require `create_launch_template` to be `true`| bool | `true` | -| eni_delete | Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) | bool | `true` | -| force\_update\_version | Force version update if existing pods are unable to be drained due to a pod disruption budget issue. | bool | Provider default behavior | -| iam\_role\_arn | IAM role ARN for workers | string | `var.default_iam_role_arn` | -| instance\_types | Node group's instance type(s). Multiple types can be specified when `capacity_type="SPOT"`. | list | `[var.workers_group_defaults[instance_type]]` | -| k8s\_labels | Kubernetes labels | map(string) | No labels applied | -| key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` | -| bootstrap_env | Provide environment variables to customise [bootstrap.sh](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh). Require `create_launch_template` to be `true` | map(string) | `{}` | -| kubelet_extra_args | Extra arguments for kubelet, this is automatically merged with `labels`. Require `create_launch_template` to be `true` | string | "" | -| launch_template_id | The id of a aws_launch_template to use | string | No LT used | -| launch\_template_version | The version of the LT to use | string | none | -| max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` | -| min\_capacity | Min number of workers | number | `var.workers_group_defaults[asg_min_size]` | -| update_config.max\_unavailable\_percentage | Max percentage of unavailable nodes during update. (e.g. 25, 50, etc) | number | `null` if `update_config.max_unavailable` is set | -| update_config.max\_unavailable | Max number of unavailable nodes during update | number | `null` if `update_config.max_unavailable_percentage` is set | -| name | Name of the node group. If you don't really need this, we recommend you to use `name_prefix` instead. | string | Will use the autogenerate name prefix | -| name_prefix | Name prefix of the node group | string | Auto generated | -| pre_userdata | userdata to pre-append to the default userdata. Require `create_launch_template` to be `true`| string | "" | -| public_ip | Associate a public ip address with a worker. Require `create_launch_template` to be `true`| string | `false` -| source\_security\_group\_ids | Source security groups for remote access to workers | list(string) | If key\_name is specified: THE REMOTE ACCESS WILL BE OPENED TO THE WORLD | -| subnets | Subnets to contain workers | list(string) | `var.workers_group_defaults[subnets]` | -| version | Kubernetes version | string | Provider default behavior | -| taints | Kubernetes node taints | list(map) | empty | -| timeouts | A map of timeouts for create/update/delete operations. | `map(string)` | Provider default behavior | -| update_default_version | Whether or not to set the new launch template version the Default | bool | `true` | -| metadata_http_endpoint | The state of the instance metadata service. Requires `create_launch_template` to be `true` | string | `var.workers_group_defaults[metadata_http_endpoint]` | -| metadata_http_tokens | If session tokens are required. Requires `create_launch_template` to be `true` | string | `var.workers_group_defaults[metadata_http_tokens]` | -| metadata_http_put_response_hop_limit | The desired HTTP PUT response hop limit for instance metadata requests. Requires `create_launch_template` to be `true` | number | `var.workers_group_defaults[metadata_http_put_response_hop_limit]` | - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.56 | -| [cloudinit](#requirement\_cloudinit) | >= 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.56 | -| [cloudinit](#provider\_cloudinit) | >= 2.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [aws_eks_node_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | -| [aws_launch_template.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | -| [cloudinit_config.workers_userdata](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of parent cluster | `string` | `""` | no | -| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of parent cluster | `string` | `""` | no | -| [cluster\_name](#input\_cluster\_name) | Name of parent cluster | `string` | `""` | no | -| [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | -| [default\_iam\_role\_arn](#input\_default\_iam\_role\_arn) | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | `""` | no | -| [ebs\_optimized\_not\_supported](#input\_ebs\_optimized\_not\_supported) | List of instance types that do not support EBS optimization | `list(string)` | `[]` | no | -| [node\_groups](#input\_node\_groups) | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | -| [node\_groups\_defaults](#input\_node\_groups\_defaults) | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | -| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | -| [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | -| [worker\_security\_group\_id](#input\_worker\_security\_group\_id) | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | -| [workers\_group\_defaults](#input\_workers\_group\_defaults) | Workers group defaults from parent | `any` | `{}` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [aws\_auth\_roles](#output\_aws\_auth\_roles) | Roles for use in aws-auth ConfigMap | -| [node\_groups](#output\_node\_groups) | Outputs from EKS node groups. Map of maps, keyed by `var.node_groups` keys. See `aws_eks_node_group` Terraform documentation for values | - diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf deleted file mode 100644 index 6abe358d5a..0000000000 --- a/modules/node_groups/launch_template.tf +++ /dev/null @@ -1,146 +0,0 @@ -data "cloudinit_config" "workers_userdata" { - for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] } - - gzip = false - base64_encode = true - boundary = "//" - - part { - content_type = "text/x-shellscript" - content = templatefile("${path.module}/templates/userdata.sh.tpl", - { - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint - cluster_auth_base64 = var.cluster_auth_base64 - ami_id = lookup(each.value, "ami_id", "") - ami_is_eks_optimized = each.value["ami_is_eks_optimized"] - bootstrap_env = each.value["bootstrap_env"] - kubelet_extra_args = each.value["kubelet_extra_args"] - pre_userdata = each.value["pre_userdata"] - capacity_type = lookup(each.value, "capacity_type", "ON_DEMAND") - append_labels = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : "" - } - ) - } -} - -# This is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) -# there are several more options one could set but you probably dont need to modify them -# you can take the default and add your custom AMI and/or custom tags -# -# Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, -# then the default user-data for bootstrapping a cluster is merged in the copy. -resource "aws_launch_template" "workers" { - for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] } - - name_prefix = local.node_groups_names[each.key] - description = format("EKS Managed Node Group custom LT for %s", local.node_groups_names[each.key]) - update_default_version = lookup(each.value, "update_default_version", true) - - block_device_mappings { - device_name = "/dev/xvda" - - ebs { - volume_size = lookup(each.value, "disk_size", null) - volume_type = lookup(each.value, "disk_type", null) - iops = lookup(each.value, "disk_iops", null) - throughput = lookup(each.value, "disk_throughput", null) - encrypted = lookup(each.value, "disk_encrypted", null) - kms_key_id = lookup(each.value, "disk_kms_key_id", null) - delete_on_termination = true - } - } - - ebs_optimized = lookup(each.value, "ebs_optimized", !contains(var.ebs_optimized_not_supported, element(each.value.instance_types, 0))) - - instance_type = each.value["set_instance_types_on_lt"] ? element(each.value.instance_types, 0) : null - - monitoring { - enabled = lookup(each.value, "enable_monitoring", null) - } - - network_interfaces { - associate_public_ip_address = lookup(each.value, "public_ip", null) - delete_on_termination = lookup(each.value, "eni_delete", null) - security_groups = compact(flatten([ - var.worker_security_group_id, - var.worker_additional_security_group_ids, - lookup( - each.value, - "additional_security_group_ids", - null, - ), - ])) - } - - # if you want to use a custom AMI - image_id = lookup(each.value, "ami_id", null) - - # If you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then - # you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 - # - # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) - - user_data = data.cloudinit_config.workers_userdata[each.key].rendered - - key_name = lookup(each.value, "key_name", null) - - metadata_options { - http_endpoint = lookup(each.value, "metadata_http_endpoint", null) - http_tokens = lookup(each.value, "metadata_http_tokens", null) - http_put_response_hop_limit = lookup(each.value, "metadata_http_put_response_hop_limit", null) - } - - # Supplying custom tags to EKS instances is another use-case for LaunchTemplates - tag_specifications { - resource_type = "instance" - - tags = merge( - var.tags, - { - Name = local.node_groups_names[each.key] - }, - lookup(var.node_groups_defaults, "additional_tags", {}), - lookup(var.node_groups[each.key], "additional_tags", {}) - ) - } - - # Supplying custom tags to EKS instances root volumes is another use-case for LaunchTemplates. (doesnt add tags to dynamically provisioned volumes via PVC tho) - tag_specifications { - resource_type = "volume" - - tags = merge( - var.tags, - { - Name = local.node_groups_names[each.key] - }, - lookup(var.node_groups_defaults, "additional_tags", {}), - lookup(var.node_groups[each.key], "additional_tags", {}) - ) - } - - # Supplying custom tags to EKS instances ENI's is another use-case for LaunchTemplates - tag_specifications { - resource_type = "network-interface" - - tags = merge( - var.tags, - { - Name = local.node_groups_names[each.key] - }, - lookup(var.node_groups_defaults, "additional_tags", {}), - lookup(var.node_groups[each.key], "additional_tags", {}) - ) - } - - # Tag the LT itself - tags = merge( - var.tags, - lookup(var.node_groups_defaults, "additional_tags", {}), - lookup(var.node_groups[each.key], "additional_tags", {}), - ) - - lifecycle { - create_before_destroy = true - } -} diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf deleted file mode 100644 index 0a6c7cbffb..0000000000 --- a/modules/node_groups/locals.tf +++ /dev/null @@ -1,51 +0,0 @@ -locals { - # Merge defaults and per-group values to make code cleaner - node_groups_expanded = { for k, v in var.node_groups : k => merge( - { - desired_capacity = var.workers_group_defaults["asg_desired_capacity"] - iam_role_arn = var.default_iam_role_arn - instance_types = [var.workers_group_defaults["instance_type"]] - key_name = var.workers_group_defaults["key_name"] - launch_template_id = var.workers_group_defaults["launch_template_id"] - launch_template_version = var.workers_group_defaults["launch_template_version"] - set_instance_types_on_lt = false - max_capacity = var.workers_group_defaults["asg_max_size"] - min_capacity = var.workers_group_defaults["asg_min_size"] - subnets = var.workers_group_defaults["subnets"] - create_launch_template = false - bootstrap_env = {} - kubelet_extra_args = var.workers_group_defaults["kubelet_extra_args"] - disk_size = var.workers_group_defaults["root_volume_size"] - disk_type = var.workers_group_defaults["root_volume_type"] - disk_iops = var.workers_group_defaults["root_iops"] - disk_throughput = var.workers_group_defaults["root_volume_throughput"] - disk_encrypted = var.workers_group_defaults["root_encrypted"] - disk_kms_key_id = var.workers_group_defaults["root_kms_key_id"] - enable_monitoring = var.workers_group_defaults["enable_monitoring"] - eni_delete = var.workers_group_defaults["eni_delete"] - public_ip = var.workers_group_defaults["public_ip"] - pre_userdata = var.workers_group_defaults["pre_userdata"] - additional_security_group_ids = var.workers_group_defaults["additional_security_group_ids"] - taints = [] - timeouts = var.workers_group_defaults["timeouts"] - update_default_version = true - ebs_optimized = null - metadata_http_endpoint = var.workers_group_defaults["metadata_http_endpoint"] - metadata_http_tokens = var.workers_group_defaults["metadata_http_tokens"] - metadata_http_put_response_hop_limit = var.workers_group_defaults["metadata_http_put_response_hop_limit"] - ami_is_eks_optimized = true - }, - var.node_groups_defaults, - v, - ) if var.create_eks } - - node_groups_names = { for k, v in local.node_groups_expanded : k => lookup( - v, - "name", - lookup( - v, - "name_prefix", - join("-", [var.cluster_name, k]) - ) - ) } -} diff --git a/modules/node_groups/main.tf b/modules/node_groups/main.tf deleted file mode 100644 index 75e6209730..0000000000 --- a/modules/node_groups/main.tf +++ /dev/null @@ -1,105 +0,0 @@ -resource "aws_eks_node_group" "workers" { - for_each = local.node_groups_expanded - - node_group_name_prefix = lookup(each.value, "name", null) == null ? local.node_groups_names[each.key] : null - node_group_name = lookup(each.value, "name", null) - - cluster_name = var.cluster_name - node_role_arn = each.value["iam_role_arn"] - subnet_ids = each.value["subnets"] - - scaling_config { - desired_size = each.value["desired_capacity"] - max_size = each.value["max_capacity"] - min_size = each.value["min_capacity"] - } - - ami_type = lookup(each.value, "ami_type", null) - disk_size = each.value["launch_template_id"] != null || each.value["create_launch_template"] ? null : lookup(each.value, "disk_size", null) - instance_types = !each.value["set_instance_types_on_lt"] ? each.value["instance_types"] : null - release_version = lookup(each.value, "ami_release_version", null) - capacity_type = lookup(each.value, "capacity_type", null) - force_update_version = lookup(each.value, "force_update_version", null) - - dynamic "remote_access" { - for_each = each.value["key_name"] != "" && each.value["launch_template_id"] == null && !each.value["create_launch_template"] ? [{ - ec2_ssh_key = each.value["key_name"] - source_security_group_ids = lookup(each.value, "source_security_group_ids", []) - }] : [] - - content { - ec2_ssh_key = remote_access.value["ec2_ssh_key"] - source_security_group_ids = remote_access.value["source_security_group_ids"] - } - } - - dynamic "launch_template" { - for_each = each.value["launch_template_id"] != null ? [{ - id = each.value["launch_template_id"] - version = each.value["launch_template_version"] - }] : [] - - content { - id = launch_template.value["id"] - version = launch_template.value["version"] - } - } - - dynamic "launch_template" { - for_each = each.value["launch_template_id"] == null && each.value["create_launch_template"] ? [{ - id = aws_launch_template.workers[each.key].id - version = each.value["launch_template_version"] == "$Latest" ? aws_launch_template.workers[each.key].latest_version : ( - each.value["launch_template_version"] == "$Default" ? aws_launch_template.workers[each.key].default_version : each.value["launch_template_version"] - ) - }] : [] - - content { - id = launch_template.value["id"] - version = launch_template.value["version"] - } - } - - dynamic "taint" { - for_each = each.value["taints"] - - content { - key = taint.value["key"] - value = taint.value["value"] - effect = taint.value["effect"] - } - } - - dynamic "update_config" { - for_each = try(each.value.update_config.max_unavailable_percentage > 0, each.value.update_config.max_unavailable > 0, false) ? [true] : [] - - content { - max_unavailable_percentage = try(each.value.update_config.max_unavailable_percentage, null) - max_unavailable = try(each.value.update_config.max_unavailable, null) - } - } - - timeouts { - create = lookup(each.value["timeouts"], "create", null) - update = lookup(each.value["timeouts"], "update", null) - delete = lookup(each.value["timeouts"], "delete", null) - } - - version = lookup(each.value, "version", null) - - labels = merge( - lookup(var.node_groups_defaults, "k8s_labels", {}), - lookup(var.node_groups[each.key], "k8s_labels", {}) - ) - - tags = merge( - var.tags, - lookup(var.node_groups_defaults, "additional_tags", {}), - lookup(var.node_groups[each.key], "additional_tags", {}), - ) - - lifecycle { - create_before_destroy = true - ignore_changes = [scaling_config[0].desired_size] - } - -} diff --git a/modules/node_groups/outputs.tf b/modules/node_groups/outputs.tf deleted file mode 100644 index ad148ea514..0000000000 --- a/modules/node_groups/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "node_groups" { - description = "Outputs from EKS node groups. Map of maps, keyed by `var.node_groups` keys. See `aws_eks_node_group` Terraform documentation for values" - value = aws_eks_node_group.workers -} - -output "aws_auth_roles" { - description = "Roles for use in aws-auth ConfigMap" - value = [ - for k, v in local.node_groups_expanded : { - worker_role_arn = lookup(v, "iam_role_arn", var.default_iam_role_arn) - platform = "linux" - } - ] -} diff --git a/modules/node_groups/templates/userdata.sh.tpl b/modules/node_groups/templates/userdata.sh.tpl deleted file mode 100644 index 321c17b427..0000000000 --- a/modules/node_groups/templates/userdata.sh.tpl +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -e -%{ if length(ami_id) == 0 ~} - -# Set bootstrap env -printf '#!/bin/bash -%{ for k, v in bootstrap_env ~} -export ${k}="${v}" -%{ endfor ~} -export ADDITIONAL_KUBELET_EXTRA_ARGS="${kubelet_extra_args}" -' > /etc/profile.d/eks-bootstrap-env.sh - -# Source extra environment variables in bootstrap script -sed -i '/^set -o errexit/a\\nsource /etc/profile.d/eks-bootstrap-env.sh' /etc/eks/bootstrap.sh - -# Merge ADDITIONAL_KUBELET_EXTRA_ARGS into KUBELET_EXTRA_ARGS -sed -i 's/^KUBELET_EXTRA_ARGS="$${KUBELET_EXTRA_ARGS:-}/KUBELET_EXTRA_ARGS="$${KUBELET_EXTRA_ARGS:-} $${ADDITIONAL_KUBELET_EXTRA_ARGS}/' /etc/eks/bootstrap.sh -%{else ~} - -# Set variables for custom AMI -API_SERVER_URL=${cluster_endpoint} -B64_CLUSTER_CA=${cluster_auth_base64} -%{ for k, v in bootstrap_env ~} -${k}="${v}" -%{ endfor ~} -KUBELET_EXTRA_ARGS='--node-labels=eks.amazonaws.com/nodegroup-image=${ami_id},eks.amazonaws.com/capacityType=${capacity_type}${append_labels} ${kubelet_extra_args}' -%{endif ~} - -# User supplied pre userdata -${pre_userdata} -%{ if length(ami_id) > 0 && ami_is_eks_optimized ~} - -# Call bootstrap for EKS optimised custom AMI -/etc/eks/bootstrap.sh ${cluster_name} --apiserver-endpoint "$${API_SERVER_URL}" --b64-cluster-ca "$${B64_CLUSTER_CA}" --kubelet-extra-args "$${KUBELET_EXTRA_ARGS}" -%{ endif ~} diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf deleted file mode 100644 index 1aa8cfe26d..0000000000 --- a/modules/node_groups/variables.tf +++ /dev/null @@ -1,71 +0,0 @@ -variable "create_eks" { - description = "Controls if EKS resources should be created (it affects almost all resources)" - type = bool - default = true -} - -variable "cluster_name" { - description = "Name of parent cluster" - type = string - default = "" -} - -variable "cluster_endpoint" { - description = "Endpoint of parent cluster" - type = string - default = "" -} - -variable "cluster_auth_base64" { - description = "Base64 encoded CA of parent cluster" - type = string - default = "" -} - -variable "default_iam_role_arn" { - description = "ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults`" - type = string - default = "" -} - -variable "workers_group_defaults" { - description = "Workers group defaults from parent" - type = any - default = {} -} - -variable "worker_security_group_id" { - description = "If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster." - type = string - default = "" -} - -variable "worker_additional_security_group_ids" { - description = "A list of additional security group ids to attach to worker instances" - type = list(string) - default = [] -} - -variable "tags" { - description = "A map of tags to add to all resources" - type = map(string) - default = {} -} - -variable "node_groups_defaults" { - description = "map of maps of node groups to create. See \"`node_groups` and `node_groups_defaults` keys\" section in README.md for more details" - type = any - default = {} -} - -variable "node_groups" { - description = "Map of maps of `eks_node_groups` to create. See \"`node_groups` and `node_groups_defaults` keys\" section in README.md for more details" - type = any - default = {} -} - -variable "ebs_optimized_not_supported" { - description = "List of instance types that do not support EBS optimization" - type = list(string) - default = [] -} diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md new file mode 100644 index 0000000000..10a3068825 --- /dev/null +++ b/modules/self-managed-node-group/README.md @@ -0,0 +1,199 @@ +# Self Managed Node Group Module + +Configuration in this directory creates a Self Managed Node Group (AutoScaling Group) along with an IAM role, security group, and launch template + +## Usage + +```hcl +module "self_managed_node_group" { + source = "terraform-aws-modules/eks/aws//modules/self-managed-node-group" + + name = "separate-self-mng" + cluster_name = "my-cluster" + cluster_version = "1.21" + cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" + cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" + + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + vpc_security_group_ids = [ + # cluster_security_group_id, + ] + + min_size = 1 + max_size = 10 + desired_size = 1 + + launch_template_name = "separate-self-mng" + instance_type = "m5.large" + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.64 | +| [cloudinit](#requirement\_cloudinit) | >= 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.64 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [user\_data](#module\_user\_data) | ../_user_data | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_autoscaling_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | +| [aws_autoscaling_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_schedule) | resource | +| [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_security_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance | `string` | `""` | no | +| [availability\_zones](#input\_availability\_zones) | A list of one or more availability zones for the group. Used for EC2-Classic and default subnets when not specified with `subnet_ids` argument. Conflicts with `subnet_ids` | `list(string)` | `null` | no | +| [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [capacity\_rebalance](#input\_capacity\_rebalance) | Indicates whether capacity rebalance is enabled | `bool` | `null` | no | +| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `null` | no | +| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | +| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | +| [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | +| [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | +| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `null` | no | +| [create](#input\_create) | Determines whether to create self managed node group or not | `bool` | `true` | no | +| [create\_iam\_instance\_profile](#input\_create\_iam\_instance\_profile) | Determines whether an IAM instance profile is created or to use an existing IAM instance profile | `bool` | `true` | no | +| [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create launch template or not | `bool` | `true` | no | +| [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | +| [create\_security\_group](#input\_create\_security\_group) | Determines whether to create a security group | `bool` | `true` | no | +| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `null` | no | +| [default\_cooldown](#input\_default\_cooldown) | The amount of time, in seconds, after a scaling activity completes before another scaling activity can start | `number` | `null` | no | +| [delete\_timeout](#input\_delete\_timeout) | Delete timeout to wait for destroying autoscaling group | `string` | `null` | no | +| [desired\_size](#input\_desired\_size) | The number of Amazon EC2 instances that should be running in the autoscaling group | `number` | `1` | no | +| [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | +| [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance will be EBS-optimized | `bool` | `null` | no | +| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `map(string)` | `null` | no | +| [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `null` | no | +| [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | +| [enabled\_metrics](#input\_enabled\_metrics) | A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances` | `list(string)` | `null` | no | +| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `null` | no | +| [force\_delete](#input\_force\_delete) | Allows deleting the Auto Scaling Group without waiting for all instances in the pool to terminate. You can force an Auto Scaling Group to delete even if it's in the process of scaling a resource. Normally, Terraform drains all the instances before deleting the group. This bypasses that behavior and potentially leaves resources dangling | `bool` | `null` | no | +| [health\_check\_grace\_period](#input\_health\_check\_grace\_period) | Time (in seconds) after instance comes into service before checking health | `number` | `null` | no | +| [health\_check\_type](#input\_health\_check\_type) | `EC2` or `ELB`. Controls how health checking is done | `string` | `null` | no | +| [hibernation\_options](#input\_hibernation\_options) | The hibernation options for the instance | `map(string)` | `null` | no | +| [iam\_instance\_profile\_arn](#input\_iam\_instance\_profile\_arn) | Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false` | `string` | `null` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | +| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | +| [initial\_lifecycle\_hooks](#input\_initial\_lifecycle\_hooks) | One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource | `list(map(string))` | `[]` | no | +| [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | +| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `null` | no | +| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated | `any` | `null` | no | +| [instance\_type](#input\_instance\_type) | The type of the instance to launch | `string` | `""` | no | +| [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | +| [key\_name](#input\_key\_name) | The key name that should be used for the instance | `string` | `null` | no | +| [launch\_template\_default\_version](#input\_launch\_template\_default\_version) | Default Version of the launch template | `string` | `null` | no | +| [launch\_template\_description](#input\_launch\_template\_description) | Description of the launch template | `string` | `null` | no | +| [launch\_template\_name](#input\_launch\_template\_name) | Launch template name - either to be created (`var.create_launch_template` = `true`) or existing (`var.create_launch_template` = `false`) | `string` | `null` | no | +| [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | +| [launch\_template\_version](#input\_launch\_template\_version) | Launch template version. Can be version number, `$Latest`, or `$Default` | `string` | `null` | no | +| [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with | `map(string)` | `null` | no | +| [max\_instance\_lifetime](#input\_max\_instance\_lifetime) | The maximum amount of time, in seconds, that an instance can be in service, values must be either equal to 0 or between 604800 and 31536000 seconds | `number` | `null` | no | +| [max\_size](#input\_max\_size) | The maximum size of the autoscaling group | `number` | `3` | no | +| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | +| [metrics\_granularity](#input\_metrics\_granularity) | The granularity to associate with the metrics to collect. The only valid value is `1Minute` | `string` | `null` | no | +| [min\_elb\_capacity](#input\_min\_elb\_capacity) | Setting this causes Terraform to wait for this number of instances to show up healthy in the ELB only on creation. Updates will not wait on ELB instance number changes | `number` | `null` | no | +| [min\_size](#input\_min\_size) | The minimum size of the autoscaling group | `number` | `0` | no | +| [mixed\_instances\_policy](#input\_mixed\_instances\_policy) | Configuration block containing settings to define launch targets for Auto Scaling groups | `any` | `null` | no | +| [name](#input\_name) | Name of the Self managed Node Group | `string` | `""` | no | +| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | +| [placement](#input\_placement) | The placement of the instance | `map(string)` | `null` | no | +| [placement\_group](#input\_placement\_group) | The name of the placement group into which you'll launch your instances, if any | `string` | `null` | no | +| [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | `string` | `"linux"` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [propagate\_tags](#input\_propagate\_tags) | A list of tag blocks. Each element should have keys named `key`, `value`, and `propagate_at_launch` | `list(map(string))` | `[]` | no | +| [protect\_from\_scale\_in](#input\_protect\_from\_scale\_in) | Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events. | `bool` | `false` | no | +| [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | +| [schedules](#input\_schedules) | Map of autoscaling group schedule to create | `map(any)` | `{}` | no | +| [security\_group\_description](#input\_security\_group\_description) | Description for the security group created | `string` | `"EKS self-managed node group security group"` | no | +| [security\_group\_name](#input\_security\_group\_name) | Name to use on security group created | `string` | `null` | no | +| [security\_group\_rules](#input\_security\_group\_rules) | List of security group rules to add to the security group created | `any` | `{}` | no | +| [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the security group created | `map(string)` | `{}` | no | +| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `string` | `true` | no | +| [service\_linked\_role\_arn](#input\_service\_linked\_role\_arn) | The ARN of the service-linked role that the ASG will use to call other AWS services | `string` | `null` | no | +| [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones` | `list(string)` | `null` | no | +| [suspended\_processes](#input\_suspended\_processes) | A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly | `list(string)` | `null` | no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | +| [target\_group\_arns](#input\_target\_group\_arns) | A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing | `list(string)` | `[]` | no | +| [termination\_policies](#input\_termination\_policies) | A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default` | `list(string)` | `null` | no | +| [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update Default Version each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | +| [use\_mixed\_instances\_policy](#input\_use\_mixed\_instances\_policy) | Determines whether to use a mixed instances policy in the autoscaling group or not | `bool` | `false` | no | +| [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix | `bool` | `true` | no | +| [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | +| [vpc\_id](#input\_vpc\_id) | ID of the VPC where the security group/nodes will be provisioned | `string` | `null` | no | +| [vpc\_security\_group\_ids](#input\_vpc\_security\_group\_ids) | A list of security group IDs to associate | `list(string)` | `[]` | no | +| [wait\_for\_capacity\_timeout](#input\_wait\_for\_capacity\_timeout) | A maximum duration that Terraform should wait for ASG instances to be healthy before timing out. (See also Waiting for Capacity below.) Setting this to '0' causes Terraform to skip all Capacity Waiting behavior. | `string` | `null` | no | +| [wait\_for\_elb\_capacity](#input\_wait\_for\_elb\_capacity) | Setting this will cause Terraform to wait for exactly this number of healthy instances in all attached load balancers on both create and update operations. Takes precedence over `min_elb_capacity` behavior. | `number` | `null` | no | +| [warm\_pool](#input\_warm\_pool) | If this block is configured, add a Warm Pool to the specified Auto Scaling group | `any` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [autoscaling\_group\_arn](#output\_autoscaling\_group\_arn) | The ARN for this autoscaling group | +| [autoscaling\_group\_availability\_zones](#output\_autoscaling\_group\_availability\_zones) | The availability zones of the autoscaling group | +| [autoscaling\_group\_default\_cooldown](#output\_autoscaling\_group\_default\_cooldown) | Time between a scaling activity and the succeeding scaling activity | +| [autoscaling\_group\_desired\_capacity](#output\_autoscaling\_group\_desired\_capacity) | The number of Amazon EC2 instances that should be running in the group | +| [autoscaling\_group\_health\_check\_grace\_period](#output\_autoscaling\_group\_health\_check\_grace\_period) | Time after instance comes into service before checking health | +| [autoscaling\_group\_health\_check\_type](#output\_autoscaling\_group\_health\_check\_type) | EC2 or ELB. Controls how health checking is done | +| [autoscaling\_group\_id](#output\_autoscaling\_group\_id) | The autoscaling group id | +| [autoscaling\_group\_max\_size](#output\_autoscaling\_group\_max\_size) | The maximum size of the autoscaling group | +| [autoscaling\_group\_min\_size](#output\_autoscaling\_group\_min\_size) | The minimum size of the autoscaling group | +| [autoscaling\_group\_name](#output\_autoscaling\_group\_name) | The autoscaling group name | +| [autoscaling\_group\_schedule\_arns](#output\_autoscaling\_group\_schedule\_arns) | ARNs of autoscaling group schedules | +| [autoscaling\_group\_vpc\_zone\_identifier](#output\_autoscaling\_group\_vpc\_zone\_identifier) | The VPC zone identifier | +| [iam\_instance\_profile\_arn](#output\_iam\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | +| [iam\_instance\_profile\_id](#output\_iam\_instance\_profile\_id) | Instance profile's ID | +| [iam\_instance\_profile\_unique](#output\_iam\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | +| [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | +| [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [launch\_template\_arn](#output\_launch\_template\_arn) | The ARN of the launch template | +| [launch\_template\_id](#output\_launch\_template\_id) | The ID of the launch template | +| [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | +| [platform](#output\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | +| [security\_group\_arn](#output\_security\_group\_arn) | Amazon Resource Name (ARN) of the security group | +| [security\_group\_id](#output\_security\_group\_id) | ID of the security group | + diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf new file mode 100644 index 0000000000..2acaa3d199 --- /dev/null +++ b/modules/self-managed-node-group/main.tf @@ -0,0 +1,554 @@ +data "aws_partition" "current" {} + +data "aws_ami" "eks_default" { + count = var.create ? 1 : 0 + + filter { + name = "name" + values = ["amazon-eks-node-${var.cluster_version}-v*"] + } + + most_recent = true + owners = ["amazon"] +} + +################################################################################ +# User Data +################################################################################ + +module "user_data" { + source = "../_user_data" + + create = var.create + platform = var.platform + is_eks_managed_node_group = false + + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + + enable_bootstrap_user_data = true + pre_bootstrap_user_data = var.pre_bootstrap_user_data + post_bootstrap_user_data = var.post_bootstrap_user_data + bootstrap_extra_args = var.bootstrap_extra_args + user_data_template_path = var.user_data_template_path +} + +################################################################################ +# Launch template +################################################################################ + +locals { + launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-node-group") +} + +resource "aws_launch_template" "this" { + count = var.create && var.create_launch_template ? 1 : 0 + + name = var.launch_template_use_name_prefix ? null : local.launch_template_name_int + name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name_int}-" : null + description = var.launch_template_description + + ebs_optimized = var.ebs_optimized + image_id = coalesce(var.ami_id, data.aws_ami.eks_default[0].image_id) + instance_type = var.instance_type + key_name = var.key_name + user_data = module.user_data.user_data + + vpc_security_group_ids = compact(concat([try(aws_security_group.this[0].id, "")], var.vpc_security_group_ids)) + + default_version = var.launch_template_default_version + update_default_version = var.update_launch_template_default_version + disable_api_termination = var.disable_api_termination + instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior + kernel_id = var.kernel_id + ram_disk_id = var.ram_disk_id + + dynamic "block_device_mappings" { + for_each = var.block_device_mappings + content { + device_name = block_device_mappings.value.device_name + no_device = lookup(block_device_mappings.value, "no_device", null) + virtual_name = lookup(block_device_mappings.value, "virtual_name", null) + + dynamic "ebs" { + for_each = flatten([lookup(block_device_mappings.value, "ebs", [])]) + content { + delete_on_termination = lookup(ebs.value, "delete_on_termination", null) + encrypted = lookup(ebs.value, "encrypted", null) + kms_key_id = lookup(ebs.value, "kms_key_id", null) + iops = lookup(ebs.value, "iops", null) + throughput = lookup(ebs.value, "throughput", null) + snapshot_id = lookup(ebs.value, "snapshot_id", null) + volume_size = lookup(ebs.value, "volume_size", null) + volume_type = lookup(ebs.value, "volume_type", null) + } + } + } + } + + dynamic "capacity_reservation_specification" { + for_each = var.capacity_reservation_specification != null ? [var.capacity_reservation_specification] : [] + content { + capacity_reservation_preference = lookup(capacity_reservation_specification.value, "capacity_reservation_preference", null) + + dynamic "capacity_reservation_target" { + for_each = lookup(capacity_reservation_specification.value, "capacity_reservation_target", []) + content { + capacity_reservation_id = lookup(capacity_reservation_target.value, "capacity_reservation_id", null) + } + } + } + } + + dynamic "cpu_options" { + for_each = var.cpu_options != null ? [var.cpu_options] : [] + content { + core_count = cpu_options.value.core_count + threads_per_core = cpu_options.value.threads_per_core + } + } + + dynamic "credit_specification" { + for_each = var.credit_specification != null ? [var.credit_specification] : [] + content { + cpu_credits = credit_specification.value.cpu_credits + } + } + + dynamic "elastic_gpu_specifications" { + for_each = var.elastic_gpu_specifications != null ? [var.elastic_gpu_specifications] : [] + content { + type = elastic_gpu_specifications.value.type + } + } + + dynamic "elastic_inference_accelerator" { + for_each = var.elastic_inference_accelerator != null ? [var.elastic_inference_accelerator] : [] + content { + type = elastic_inference_accelerator.value.type + } + } + + dynamic "enclave_options" { + for_each = var.enclave_options != null ? [var.enclave_options] : [] + content { + enabled = enclave_options.value.enabled + } + } + + dynamic "hibernation_options" { + for_each = var.hibernation_options != null ? [var.hibernation_options] : [] + content { + configured = hibernation_options.value.configured + } + } + + iam_instance_profile { + arn = var.create_iam_instance_profile ? aws_iam_instance_profile.this[0].arn : var.iam_instance_profile_arn + } + + dynamic "instance_market_options" { + for_each = var.instance_market_options != null ? [var.instance_market_options] : [] + content { + market_type = instance_market_options.value.market_type + + dynamic "spot_options" { + for_each = lookup(instance_market_options.value, "spot_options", null) != null ? [instance_market_options.value.spot_options] : [] + content { + block_duration_minutes = spot_options.value.block_duration_minutes + instance_interruption_behavior = lookup(spot_options.value, "instance_interruption_behavior", null) + max_price = lookup(spot_options.value, "max_price", null) + spot_instance_type = lookup(spot_options.value, "spot_instance_type", null) + valid_until = lookup(spot_options.value, "valid_until", null) + } + } + } + } + + dynamic "license_specification" { + for_each = var.license_specifications != null ? [var.license_specifications] : [] + content { + license_configuration_arn = license_specifications.value.license_configuration_arn + } + } + + dynamic "metadata_options" { + for_each = var.metadata_options != null ? [var.metadata_options] : [] + content { + http_endpoint = lookup(metadata_options.value, "http_endpoint", null) + http_tokens = lookup(metadata_options.value, "http_tokens", null) + http_put_response_hop_limit = lookup(metadata_options.value, "http_put_response_hop_limit", null) + http_protocol_ipv6 = lookup(metadata_options.value, "http_protocol_ipv6", null) + } + } + + dynamic "monitoring" { + for_each = var.enable_monitoring != null ? [1] : [] + content { + enabled = var.enable_monitoring + } + } + + dynamic "network_interfaces" { + for_each = var.network_interfaces + content { + associate_carrier_ip_address = lookup(network_interfaces.value, "associate_carrier_ip_address", null) + associate_public_ip_address = lookup(network_interfaces.value, "associate_public_ip_address", null) + delete_on_termination = lookup(network_interfaces.value, "delete_on_termination", null) + description = lookup(network_interfaces.value, "description", null) + device_index = lookup(network_interfaces.value, "device_index", null) + ipv4_addresses = lookup(network_interfaces.value, "ipv4_addresses", null) != null ? network_interfaces.value.ipv4_addresses : [] + ipv4_address_count = lookup(network_interfaces.value, "ipv4_address_count", null) + ipv6_addresses = lookup(network_interfaces.value, "ipv6_addresses", null) != null ? network_interfaces.value.ipv6_addresses : [] + ipv6_address_count = lookup(network_interfaces.value, "ipv6_address_count", null) + network_interface_id = lookup(network_interfaces.value, "network_interface_id", null) + private_ip_address = lookup(network_interfaces.value, "private_ip_address", null) + security_groups = lookup(network_interfaces.value, "security_groups", null) != null ? network_interfaces.value.security_groups : [] + subnet_id = lookup(network_interfaces.value, "subnet_id", null) + } + } + + dynamic "placement" { + for_each = var.placement != null ? [var.placement] : [] + content { + affinity = lookup(placement.value, "affinity", null) + availability_zone = lookup(placement.value, "availability_zone", null) + group_name = lookup(placement.value, "group_name", null) + host_id = lookup(placement.value, "host_id", null) + spread_domain = lookup(placement.value, "spread_domain", null) + tenancy = lookup(placement.value, "tenancy", null) + partition_number = lookup(placement.value, "partition_number", null) + } + } + + dynamic "tag_specifications" { + for_each = toset(["instance", "volume", "network-interface"]) + content { + resource_type = tag_specifications.key + tags = merge(var.tags, { Name = var.name }) + } + } + + lifecycle { + create_before_destroy = true + } + + # Prevent premature access of security group roles and policies by pods that + # require permissions on create/destroy that depend on nodes + depends_on = [ + aws_security_group_rule.this, + aws_iam_role_policy_attachment.this, + ] + + tags = var.tags +} + +################################################################################ +# Node Group +################################################################################ + +locals { + launch_template_name = try(aws_launch_template.this[0].name, var.launch_template_name) + # Change order to allow users to set version priority before using defaults + launch_template_version = coalesce(var.launch_template_version, try(aws_launch_template.this[0].default_version, "$Default")) +} + +resource "aws_autoscaling_group" "this" { + count = var.create ? 1 : 0 + + name = var.use_name_prefix ? null : var.name + name_prefix = var.use_name_prefix ? "${var.name}-" : null + + dynamic "launch_template" { + for_each = var.use_mixed_instances_policy ? [] : [1] + + content { + name = local.launch_template_name + version = local.launch_template_version + } + } + + availability_zones = var.availability_zones + vpc_zone_identifier = var.subnet_ids + + min_size = var.min_size + max_size = var.max_size + desired_capacity = var.desired_size + capacity_rebalance = var.capacity_rebalance + min_elb_capacity = var.min_elb_capacity + wait_for_elb_capacity = var.wait_for_elb_capacity + wait_for_capacity_timeout = var.wait_for_capacity_timeout + default_cooldown = var.default_cooldown + protect_from_scale_in = var.protect_from_scale_in + + target_group_arns = var.target_group_arns + placement_group = var.placement_group + health_check_type = var.health_check_type + health_check_grace_period = var.health_check_grace_period + + force_delete = var.force_delete + termination_policies = var.termination_policies + suspended_processes = var.suspended_processes + max_instance_lifetime = var.max_instance_lifetime + + enabled_metrics = var.enabled_metrics + metrics_granularity = var.metrics_granularity + service_linked_role_arn = var.service_linked_role_arn + + dynamic "initial_lifecycle_hook" { + for_each = var.initial_lifecycle_hooks + content { + name = initial_lifecycle_hook.value.name + default_result = lookup(initial_lifecycle_hook.value, "default_result", null) + heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) + lifecycle_transition = initial_lifecycle_hook.value.lifecycle_transition + notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) + notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) + role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) + } + } + + dynamic "instance_refresh" { + for_each = var.instance_refresh != null ? [var.instance_refresh] : [] + content { + strategy = instance_refresh.value.strategy + triggers = lookup(instance_refresh.value, "triggers", null) + + dynamic "preferences" { + for_each = lookup(instance_refresh.value, "preferences", null) != null ? [instance_refresh.value.preferences] : [] + content { + instance_warmup = lookup(preferences.value, "instance_warmup", null) + min_healthy_percentage = lookup(preferences.value, "min_healthy_percentage", null) + checkpoint_delay = lookup(preferences.value, "checkpoint_delay", null) + checkpoint_percentages = lookup(preferences.value, "checkpoint_percentages", null) + } + } + } + } + + dynamic "mixed_instances_policy" { + for_each = var.use_mixed_instances_policy ? [var.mixed_instances_policy] : [] + content { + dynamic "instances_distribution" { + for_each = try([mixed_instances_policy.value.instances_distribution], []) + content { + on_demand_allocation_strategy = lookup(instances_distribution.value, "on_demand_allocation_strategy", null) + on_demand_base_capacity = lookup(instances_distribution.value, "on_demand_base_capacity", null) + on_demand_percentage_above_base_capacity = lookup(instances_distribution.value, "on_demand_percentage_above_base_capacity", null) + spot_allocation_strategy = lookup(instances_distribution.value, "spot_allocation_strategy", null) + spot_instance_pools = lookup(instances_distribution.value, "spot_instance_pools", null) + spot_max_price = lookup(instances_distribution.value, "spot_max_price", null) + } + } + + launch_template { + launch_template_specification { + launch_template_name = local.launch_template_name + version = local.launch_template_version + } + + dynamic "override" { + for_each = try(mixed_instances_policy.value.override, []) + content { + instance_type = lookup(override.value, "instance_type", null) + weighted_capacity = lookup(override.value, "weighted_capacity", null) + + dynamic "launch_template_specification" { + for_each = lookup(override.value, "launch_template_specification", null) != null ? override.value.launch_template_specification : [] + content { + launch_template_id = lookup(launch_template_specification.value, "launch_template_id", null) + } + } + } + } + } + } + } + + dynamic "warm_pool" { + for_each = var.warm_pool != null ? [var.warm_pool] : [] + content { + pool_state = lookup(warm_pool.value, "pool_state", null) + min_size = lookup(warm_pool.value, "min_size", null) + max_group_prepared_capacity = lookup(warm_pool.value, "max_group_prepared_capacity", null) + } + } + + timeouts { + delete = var.delete_timeout + } + + lifecycle { + create_before_destroy = true + ignore_changes = [ + desired_capacity + ] + } + + tags = concat( + [ + { + key = "Name" + value = var.name + propagate_at_launch = true + }, + { + key = "kubernetes.io/cluster/${var.cluster_name}" + value = "owned" + propagate_at_launch = true + }, + { + key = "k8s.io/cluster/${var.cluster_name}" + value = "owned" + propagate_at_launch = true + }, + ], + var.propagate_tags, + [for k, v in var.tags : + { + key = k + value = v + propagate_at_launch = true + } + ] + ) +} + +################################################################################ +# Autoscaling group schedule +################################################################################ + +resource "aws_autoscaling_schedule" "this" { + for_each = var.create && var.create_schedule ? var.schedules : {} + + scheduled_action_name = each.key + autoscaling_group_name = aws_autoscaling_group.this[0].name + + min_size = lookup(each.value, "min_size", null) + max_size = lookup(each.value, "max_size", null) + desired_capacity = lookup(each.value, "desired_size", null) + start_time = lookup(each.value, "start_time", null) + end_time = lookup(each.value, "end_time", null) + time_zone = lookup(each.value, "time_zone", null) + + # [Minute] [Hour] [Day_of_Month] [Month_of_Year] [Day_of_Week] + # Cron examples: https://crontab.guru/examples.html + recurrence = lookup(each.value, "recurrence", null) +} + +################################################################################ +# Security Group +################################################################################ + +locals { + security_group_name = coalesce(var.security_group_name, "${var.name}-node-group") + create_security_group = var.create && var.create_security_group +} + +resource "aws_security_group" "this" { + count = local.create_security_group ? 1 : 0 + + name = var.security_group_use_name_prefix ? null : local.security_group_name + name_prefix = var.security_group_use_name_prefix ? "${local.security_group_name}-" : null + description = var.security_group_description + vpc_id = var.vpc_id + + tags = merge( + var.tags, + { + "Name" = local.security_group_name + "kubernetes.io/cluster/${var.cluster_name}" = "owned" + }, + var.security_group_tags + ) +} + +resource "aws_security_group_rule" "this" { + for_each = { for k, v in var.security_group_rules : k => v if local.create_security_group } + + # Required + security_group_id = aws_security_group.this[0].id + protocol = each.value.protocol + from_port = each.value.from_port + to_port = each.value.to_port + type = each.value.type + + # Optional + description = try(each.value.description, null) + cidr_blocks = try(each.value.cidr_blocks, null) + ipv6_cidr_blocks = try(each.value.ipv6_cidr_blocks, null) + prefix_list_ids = try(each.value.prefix_list_ids, []) + self = try(each.value.self, null) + source_security_group_id = try( + each.value.source_security_group_id, + try(each.value.source_cluster_security_group, false) ? var.cluster_security_group_id : null + ) +} + +################################################################################ +# IAM Role +################################################################################ + +locals { + iam_role_name = coalesce(var.iam_role_name, "${var.name}-node-group") + + iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" +} + +data "aws_iam_policy_document" "assume_role_policy" { + count = var.create && var.create_iam_instance_profile ? 1 : 0 + + statement { + sid = "EKSNodeAssumeRole" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.${data.aws_partition.current.dns_suffix}"] + } + } +} + +resource "aws_iam_role" "this" { + count = var.create && var.create_iam_instance_profile ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + assume_role_policy = data.aws_iam_policy_document.assume_role_policy[0].json + permissions_boundary = var.iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +resource "aws_iam_role_policy_attachment" "this" { + for_each = var.create && var.create_iam_instance_profile ? toset(compact(distinct(concat([ + "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", + "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", + var.iam_role_attach_cni_policy ? "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" : "", + ], var.iam_role_additional_policies)))) : toset([]) + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +# Only self-managed node group requires instance profile +resource "aws_iam_instance_profile" "this" { + count = var.create && var.create_iam_instance_profile ? 1 : 0 + + role = aws_iam_role.this[0].name + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + + lifecycle { + create_before_destroy = true + } + + tags = merge(var.tags, var.iam_role_tags) +} diff --git a/modules/self-managed-node-group/outputs.tf b/modules/self-managed-node-group/outputs.tf new file mode 100644 index 0000000000..983c92a80c --- /dev/null +++ b/modules/self-managed-node-group/outputs.tf @@ -0,0 +1,147 @@ +################################################################################ +# Launch template +################################################################################ + +output "launch_template_id" { + description = "The ID of the launch template" + value = try(aws_launch_template.this[0].id, "") +} + +output "launch_template_arn" { + description = "The ARN of the launch template" + value = try(aws_launch_template.this[0].arn, "") +} + +output "launch_template_latest_version" { + description = "The latest version of the launch template" + value = try(aws_launch_template.this[0].latest_version, "") +} + +################################################################################ +# autoscaling group +################################################################################ + +output "autoscaling_group_arn" { + description = "The ARN for this autoscaling group" + value = try(aws_autoscaling_group.this[0].arn, "") +} + +output "autoscaling_group_id" { + description = "The autoscaling group id" + value = try(aws_autoscaling_group.this[0].id, "") +} + +output "autoscaling_group_name" { + description = "The autoscaling group name" + value = try(aws_autoscaling_group.this[0].name, "") +} + +output "autoscaling_group_min_size" { + description = "The minimum size of the autoscaling group" + value = try(aws_autoscaling_group.this[0].min_size, "") +} + +output "autoscaling_group_max_size" { + description = "The maximum size of the autoscaling group" + value = try(aws_autoscaling_group.this[0].max_size, "") +} + +output "autoscaling_group_desired_capacity" { + description = "The number of Amazon EC2 instances that should be running in the group" + value = try(aws_autoscaling_group.this[0].desired_capacity, "") +} + +output "autoscaling_group_default_cooldown" { + description = "Time between a scaling activity and the succeeding scaling activity" + value = try(aws_autoscaling_group.this[0].default_cooldown, "") +} + +output "autoscaling_group_health_check_grace_period" { + description = "Time after instance comes into service before checking health" + value = try(aws_autoscaling_group.this[0].health_check_grace_period, "") +} + +output "autoscaling_group_health_check_type" { + description = "EC2 or ELB. Controls how health checking is done" + value = try(aws_autoscaling_group.this[0].health_check_type, "") +} + +output "autoscaling_group_availability_zones" { + description = "The availability zones of the autoscaling group" + value = try(aws_autoscaling_group.this[0].availability_zones, "") +} + +output "autoscaling_group_vpc_zone_identifier" { + description = "The VPC zone identifier" + value = try(aws_autoscaling_group.this[0].vpc_zone_identifier, "") +} + +################################################################################ +# autoscaling group schedule +################################################################################ + +output "autoscaling_group_schedule_arns" { + description = "ARNs of autoscaling group schedules" + value = { for k, v in aws_autoscaling_schedule.this : k => v.arn } +} + +################################################################################ +# Security Group +################################################################################ + +output "security_group_arn" { + description = "Amazon Resource Name (ARN) of the security group" + value = try(aws_security_group.this[0].arn, "") +} + +output "security_group_id" { + description = "ID of the security group" + value = try(aws_security_group.this[0].id, "") +} + +################################################################################ +# IAM Role +################################################################################ + +output "iam_role_name" { + description = "The name of the IAM role" + value = try(aws_iam_role.this[0].name, "") +} + +output "iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = try(aws_iam_role.this[0].arn, "") +} + +output "iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, "") +} + +################################################################################ +# IAM Instance Profile +################################################################################ + +output "iam_instance_profile_arn" { + description = "ARN assigned by AWS to the instance profile" + value = try(aws_iam_instance_profile.this[0].arn, "") +} + +output "iam_instance_profile_id" { + description = "Instance profile's ID" + value = try(aws_iam_instance_profile.this[0].id, "") +} + +output "iam_instance_profile_unique" { + description = "Stable and unique string identifying the IAM instance profile" + value = try(aws_iam_instance_profile.this[0].unique_id, "") +} + +################################################################################ +# Additional +################################################################################ + +output "platform" { + description = "Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based" + value = var.platform +} diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf new file mode 100644 index 0000000000..d5fe48dec5 --- /dev/null +++ b/modules/self-managed-node-group/variables.tf @@ -0,0 +1,579 @@ +variable "create" { + description = "Determines whether to create self managed node group or not" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +variable "platform" { + description = "Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based" + type = string + default = "linux" +} + +################################################################################ +# User Data +################################################################################ + +variable "cluster_name" { + description = "Name of associated EKS cluster" + type = string + default = null +} + +variable "cluster_endpoint" { + description = "Endpoint of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_auth_base64" { + description = "Base64 encoded CA of associated EKS cluster" + type = string + default = "" +} + +variable "pre_bootstrap_user_data" { + description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + type = string + default = "" +} + +variable "post_bootstrap_user_data" { + description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + type = string + default = "" +} + +variable "bootstrap_extra_args" { + description = "Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" + type = string + default = "" +} + +variable "user_data_template_path" { + description = "Path to a local, custom user data template file to use when rendering user data" + type = string + default = "" +} + +################################################################################ +# Launch template +################################################################################ + +variable "create_launch_template" { + description = "Determines whether to create launch template or not" + type = bool + default = true +} + +variable "launch_template_name" { + description = "Launch template name - either to be created (`var.create_launch_template` = `true`) or existing (`var.create_launch_template` = `false`)" + type = string + default = null +} + +variable "launch_template_use_name_prefix" { + description = "Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix" + type = bool + default = true +} + +variable "launch_template_description" { + description = "Description of the launch template" + type = string + default = null +} + +variable "launch_template_default_version" { + description = "Default Version of the launch template" + type = string + default = null +} + +variable "update_launch_template_default_version" { + description = "Whether to update Default Version each update. Conflicts with `launch_template_default_version`" + type = bool + default = true +} + +variable "disable_api_termination" { + description = "If true, enables EC2 instance termination protection" + type = bool + default = null +} + +variable "instance_initiated_shutdown_behavior" { + description = "Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`)" + type = string + default = null +} + +variable "kernel_id" { + description = "The kernel ID" + type = string + default = null +} + +variable "ram_disk_id" { + description = "The ID of the ram disk" + type = string + default = null +} + +variable "block_device_mappings" { + description = "Specify volumes to attach to the instance besides the volumes specified by the AMI" + type = any + default = {} +} + +variable "capacity_reservation_specification" { + description = "Targeting for EC2 capacity reservations" + type = any + default = null +} + +variable "cpu_options" { + description = "The CPU options for the instance" + type = map(string) + default = null +} + +variable "credit_specification" { + description = "Customize the credit specification of the instance" + type = map(string) + default = null +} + +variable "elastic_gpu_specifications" { + description = "The elastic GPU to attach to the instance" + type = map(string) + default = null +} + +variable "elastic_inference_accelerator" { + description = "Configuration block containing an Elastic Inference Accelerator to attach to the instance" + type = map(string) + default = null +} + +variable "enclave_options" { + description = "Enable Nitro Enclaves on launched instances" + type = map(string) + default = null +} + +variable "hibernation_options" { + description = "The hibernation options for the instance" + type = map(string) + default = null +} + +variable "instance_market_options" { + description = "The market (purchasing) option for the instance" + type = any + default = null +} + +variable "license_specifications" { + description = "A list of license specifications to associate with" + type = map(string) + default = null +} + +variable "network_interfaces" { + description = "Customize network interfaces to be attached at instance boot time" + type = list(any) + default = [] +} + +variable "placement" { + description = "The placement of the instance" + type = map(string) + default = null +} + +variable "ebs_optimized" { + description = "If true, the launched EC2 instance will be EBS-optimized" + type = bool + default = null +} + +variable "ami_id" { + description = "The AMI from which to launch the instance" + type = string + default = "" +} + +variable "cluster_version" { + description = "Kubernetes cluster version - used to lookup default AMI ID if one is not provided" + type = string + default = null +} + +variable "instance_type" { + description = "The type of the instance to launch" + type = string + default = "" +} + +variable "key_name" { + description = "The key name that should be used for the instance" + type = string + default = null +} + +variable "vpc_security_group_ids" { + description = "A list of security group IDs to associate" + type = list(string) + default = [] +} + +variable "enable_monitoring" { + description = "Enables/disables detailed monitoring" + type = bool + default = true +} + +variable "metadata_options" { + description = "Customize the metadata options for the instance" + type = map(string) + default = { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 2 + } +} + +################################################################################ +# Autoscaling group +################################################################################ + +variable "name" { + description = "Name of the Self managed Node Group" + type = string + default = "" +} + +variable "use_name_prefix" { + description = "Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix" + type = bool + default = true +} + +variable "launch_template_version" { + description = "Launch template version. Can be version number, `$Latest`, or `$Default`" + type = string + default = null +} + +variable "availability_zones" { + description = "A list of one or more availability zones for the group. Used for EC2-Classic and default subnets when not specified with `subnet_ids` argument. Conflicts with `subnet_ids`" + type = list(string) + default = null +} + +variable "subnet_ids" { + description = "A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones`" + type = list(string) + default = null +} + +variable "min_size" { + description = "The minimum size of the autoscaling group" + type = number + default = 0 +} + +variable "max_size" { + description = "The maximum size of the autoscaling group" + type = number + default = 3 +} + +variable "desired_size" { + description = "The number of Amazon EC2 instances that should be running in the autoscaling group" + type = number + default = 1 +} + +variable "capacity_rebalance" { + description = "Indicates whether capacity rebalance is enabled" + type = bool + default = null +} + +variable "min_elb_capacity" { + description = "Setting this causes Terraform to wait for this number of instances to show up healthy in the ELB only on creation. Updates will not wait on ELB instance number changes" + type = number + default = null +} + +variable "wait_for_elb_capacity" { + description = "Setting this will cause Terraform to wait for exactly this number of healthy instances in all attached load balancers on both create and update operations. Takes precedence over `min_elb_capacity` behavior." + type = number + default = null +} + +variable "wait_for_capacity_timeout" { + description = "A maximum duration that Terraform should wait for ASG instances to be healthy before timing out. (See also Waiting for Capacity below.) Setting this to '0' causes Terraform to skip all Capacity Waiting behavior." + type = string + default = null +} + +variable "default_cooldown" { + description = "The amount of time, in seconds, after a scaling activity completes before another scaling activity can start" + type = number + default = null +} + +variable "protect_from_scale_in" { + description = "Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events." + type = bool + default = false +} + +variable "target_group_arns" { + description = "A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing" + type = list(string) + default = [] +} + +variable "placement_group" { + description = "The name of the placement group into which you'll launch your instances, if any" + type = string + default = null +} + +variable "health_check_type" { + description = "`EC2` or `ELB`. Controls how health checking is done" + type = string + default = null +} + +variable "health_check_grace_period" { + description = "Time (in seconds) after instance comes into service before checking health" + type = number + default = null +} + +variable "force_delete" { + description = "Allows deleting the Auto Scaling Group without waiting for all instances in the pool to terminate. You can force an Auto Scaling Group to delete even if it's in the process of scaling a resource. Normally, Terraform drains all the instances before deleting the group. This bypasses that behavior and potentially leaves resources dangling" + type = bool + default = null +} + +variable "termination_policies" { + description = "A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default`" + type = list(string) + default = null +} + +variable "suspended_processes" { + description = "A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly" + type = list(string) + default = null +} + +variable "max_instance_lifetime" { + description = "The maximum amount of time, in seconds, that an instance can be in service, values must be either equal to 0 or between 604800 and 31536000 seconds" + type = number + default = null +} + +variable "enabled_metrics" { + description = "A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances`" + type = list(string) + default = null +} + +variable "metrics_granularity" { + description = "The granularity to associate with the metrics to collect. The only valid value is `1Minute`" + type = string + default = null +} + +variable "service_linked_role_arn" { + description = "The ARN of the service-linked role that the ASG will use to call other AWS services" + type = string + default = null +} + +variable "initial_lifecycle_hooks" { + description = "One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource" + type = list(map(string)) + default = [] +} + +variable "instance_refresh" { + description = "If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated" + type = any + default = null +} + +variable "use_mixed_instances_policy" { + description = "Determines whether to use a mixed instances policy in the autoscaling group or not" + type = bool + default = false +} + +variable "mixed_instances_policy" { + description = "Configuration block containing settings to define launch targets for Auto Scaling groups" + type = any + default = null +} + +variable "warm_pool" { + description = "If this block is configured, add a Warm Pool to the specified Auto Scaling group" + type = any + default = null +} + +variable "delete_timeout" { + description = "Delete timeout to wait for destroying autoscaling group" + type = string + default = null +} + +variable "propagate_tags" { + description = "A list of tag blocks. Each element should have keys named `key`, `value`, and `propagate_at_launch`" + type = list(map(string)) + default = [] +} + +################################################################################ +# Autoscaling group schedule +################################################################################ + +variable "create_schedule" { + description = "Determines whether to create autoscaling group schedule or not" + type = bool + default = true +} + +variable "schedules" { + description = "Map of autoscaling group schedule to create" + type = map(any) + default = {} +} + +################################################################################ +# Security Group +################################################################################ + +variable "create_security_group" { + description = "Determines whether to create a security group" + type = bool + default = true +} + +variable "security_group_name" { + description = "Name to use on security group created" + type = string + default = null +} + +variable "security_group_use_name_prefix" { + description = "Determines whether the security group name (`security_group_name`) is used as a prefix" + type = string + default = true +} + +variable "security_group_description" { + description = "Description for the security group created" + type = string + default = "EKS self-managed node group security group" +} + +variable "vpc_id" { + description = "ID of the VPC where the security group/nodes will be provisioned" + type = string + default = null +} + +variable "security_group_rules" { + description = "List of security group rules to add to the security group created" + type = any + default = {} +} + +variable "cluster_security_group_id" { + description = "Cluster control plane security group ID" + type = string + default = null +} + +variable "security_group_tags" { + description = "A map of additional tags to add to the security group created" + type = map(string) + default = {} +} + +################################################################################ +# IAM Role +################################################################################ + +variable "create_iam_instance_profile" { + description = "Determines whether an IAM instance profile is created or to use an existing IAM instance profile" + type = bool + default = true +} + +variable "iam_instance_profile_arn" { + description = "Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false`" + type = string + default = null +} + +variable "iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix" + type = string + default = true +} + +variable "iam_role_path" { + description = "IAM role path" + type = string + default = null +} + +variable "iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "iam_role_attach_cni_policy" { + description = "Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" + type = bool + default = true +} + +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = list(string) + default = [] +} + +variable "iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf new file mode 100644 index 0000000000..48492037e2 --- /dev/null +++ b/modules/self-managed-node-group/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.64" + } + cloudinit = { + source = "hashicorp/cloudinit" + version = ">= 2.0" + } + } +} diff --git a/node_groups.tf b/node_groups.tf index 531a3df480..771fa60bf8 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -1,27 +1,397 @@ -module "node_groups" { - source = "./modules/node_groups" +locals { + metadata_options = { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 2 + } +} + +################################################################################ +# Node Security Group +# Defaults follow https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html +# Plus NTP/HTTPS (otherwise nodes fail to launch) +################################################################################ + +locals { + node_sg_name = coalesce(var.node_security_group_name, "${var.cluster_name}-node") + create_node_sg = var.create && var.create_node_security_group + + node_security_group_id = local.create_node_sg ? aws_security_group.node[0].id : var.node_security_group_id + + node_security_group_rules = { + egress_cluster_443 = { + description = "Node groups to cluster API" + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "egress" + source_cluster_security_group = true + } + ingress_cluster_443 = { + description = "Cluster API to node groups" + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "ingress" + source_cluster_security_group = true + } + ingress_cluster_kubelet = { + description = "Cluster API to node kubelets" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + type = "ingress" + source_cluster_security_group = true + } + ingress_self_coredns_tcp = { + description = "Node to node CoreDNS" + protocol = "tcp" + from_port = 53 + to_port = 53 + type = "ingress" + self = true + } + egress_self_coredns_tcp = { + description = "Node to node CoreDNS" + protocol = "tcp" + from_port = 53 + to_port = 53 + type = "egress" + self = true + } + ingress_self_coredns_udp = { + description = "Node to node CoreDNS" + protocol = "udp" + from_port = 53 + to_port = 53 + type = "ingress" + self = true + } + egress_self_coredns_udp = { + description = "Node to node CoreDNS" + protocol = "udp" + from_port = 53 + to_port = 53 + type = "egress" + self = true + } + egress_https = { + description = "Egress all HTTPS to internet" + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + } + egress_ntp_tcp = { + description = "Egress NTP/TCP to internet" + protocol = "tcp" + from_port = 123 + to_port = 123 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + } + egress_ntp_udp = { + description = "Egress NTP/UDP to internet" + protocol = "udp" + from_port = 123 + to_port = 123 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + } + } +} + +resource "aws_security_group" "node" { + count = local.create_node_sg ? 1 : 0 + + name = var.node_security_group_use_name_prefix ? null : local.node_sg_name + name_prefix = var.node_security_group_use_name_prefix ? "${local.node_sg_name}-" : null + description = var.node_security_group_description + vpc_id = var.vpc_id + + tags = merge( + var.tags, + { + "Name" = local.node_sg_name + "kubernetes.io/cluster/${var.cluster_name}" = "owned" + }, + var.node_security_group_tags + ) +} + +resource "aws_security_group_rule" "node" { + for_each = { for k, v in merge(local.node_security_group_rules, var.node_security_group_additional_rules) : k => v if local.create_node_sg } + + # Required + security_group_id = aws_security_group.node[0].id + protocol = each.value.protocol + from_port = each.value.from_port + to_port = each.value.to_port + type = each.value.type + + # Optional + description = try(each.value.description, null) + cidr_blocks = try(each.value.cidr_blocks, null) + ipv6_cidr_blocks = try(each.value.ipv6_cidr_blocks, null) + prefix_list_ids = try(each.value.prefix_list_ids, []) + self = try(each.value.self, null) + source_security_group_id = try( + each.value.source_security_group_id, + try(each.value.source_cluster_security_group, false) ? local.cluster_security_group_id : null + ) +} + +################################################################################ +# Fargate Profile +################################################################################ + +module "fargate_profile" { + source = "./modules/fargate-profile" + + for_each = { for k, v in var.fargate_profiles : k => v if var.create } + + # Fargate Profile + cluster_name = aws_eks_cluster.this[0].name + name = try(each.value.name, each.key) + subnet_ids = try(each.value.subnet_ids, var.fargate_profile_defaults.subnet_ids, var.subnet_ids) + selectors = try(each.value.selectors, var.fargate_profile_defaults.selectors, []) + timeouts = try(each.value.timeouts, var.fargate_profile_defaults.timeouts, {}) + + # IAM role + create_iam_role = try(each.value.create_iam_role, var.fargate_profile_defaults.create_iam_role, true) + iam_role_arn = try(each.value.iam_role_arn, var.fargate_profile_defaults.iam_role_arn, null) + iam_role_name = try(each.value.iam_role_name, var.fargate_profile_defaults.iam_role_name, null) + iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, var.fargate_profile_defaults.iam_role_use_name_prefix, true) + iam_role_path = try(each.value.iam_role_path, var.fargate_profile_defaults.iam_role_path, null) + iam_role_description = try(each.value.iam_role_description, var.fargate_profile_defaults.iam_role_description, "Fargate profile IAM role") + iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.fargate_profile_defaults.iam_role_permissions_boundary, null) + iam_role_tags = try(each.value.iam_role_tags, var.fargate_profile_defaults.iam_role_tags, {}) + iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.fargate_profile_defaults.iam_role_additional_policies, []) + + tags = merge(var.tags, try(each.value.tags, var.fargate_profile_defaults.tags, {})) +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +module "eks_managed_node_group" { + source = "./modules/eks-managed-node-group" + + for_each = { for k, v in var.eks_managed_node_groups : k => v if var.create } + + cluster_name = aws_eks_cluster.this[0].name + cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, var.cluster_version) + cluster_security_group_id = local.cluster_security_group_id + + # EKS Managed Node Group + name = try(each.value.name, each.key) + use_name_prefix = try(each.value.use_name_prefix, var.eks_managed_node_group_defaults.use_name_prefix, true) + + subnet_ids = try(each.value.subnet_ids, var.eks_managed_node_group_defaults.subnet_ids, var.subnet_ids) + + min_size = try(each.value.min_size, var.eks_managed_node_group_defaults.min_size, 1) + max_size = try(each.value.max_size, var.eks_managed_node_group_defaults.max_size, 3) + desired_size = try(each.value.desired_size, var.eks_managed_node_group_defaults.desired_size, 1) + + ami_id = try(each.value.ami_id, var.eks_managed_node_group_defaults.ami_id, "") + ami_type = try(each.value.ami_type, var.eks_managed_node_group_defaults.ami_type, null) + ami_release_version = try(each.value.ami_release_version, var.eks_managed_node_group_defaults.ami_release_version, null) + + capacity_type = try(each.value.capacity_type, var.eks_managed_node_group_defaults.capacity_type, null) + disk_size = try(each.value.disk_size, var.eks_managed_node_group_defaults.disk_size, null) + force_update_version = try(each.value.force_update_version, var.eks_managed_node_group_defaults.force_update_version, null) + instance_types = try(each.value.instance_types, var.eks_managed_node_group_defaults.instance_types, null) + labels = try(each.value.labels, var.eks_managed_node_group_defaults.labels, null) + + remote_access = try(each.value.remote_access, var.eks_managed_node_group_defaults.remote_access, {}) + taints = try(each.value.taints, var.eks_managed_node_group_defaults.taints, {}) + update_config = try(each.value.update_config, var.eks_managed_node_group_defaults.update_config, {}) + timeouts = try(each.value.timeouts, var.eks_managed_node_group_defaults.timeouts, {}) + + # User data + platform = try(each.value.platform, var.eks_managed_node_group_defaults.platform, "linux") + cluster_endpoint = try(aws_eks_cluster.this[0].endpoint, "") + cluster_auth_base64 = try(aws_eks_cluster.this[0].certificate_authority[0].data, "") + cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr + enable_bootstrap_user_data = try(each.value.enable_bootstrap_user_data, var.eks_managed_node_group_defaults.enable_bootstrap_user_data, false) + pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.eks_managed_node_group_defaults.pre_bootstrap_user_data, "") + post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.eks_managed_node_group_defaults.post_bootstrap_user_data, "") + bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.eks_managed_node_group_defaults.bootstrap_extra_args, "") + user_data_template_path = try(each.value.user_data_template_path, var.eks_managed_node_group_defaults.user_data_template_path, "") + + # Launch Template + create_launch_template = try(each.value.create_launch_template, var.eks_managed_node_group_defaults.create_launch_template, true) + launch_template_name = try(each.value.launch_template_name, each.key) + launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.eks_managed_node_group_defaults.launch_template_use_name_prefix, true) + launch_template_version = try(each.value.launch_template_version, var.eks_managed_node_group_defaults.launch_template_version, null) + launch_template_description = try(each.value.launch_template_description, var.eks_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} EKS managed node group") + + ebs_optimized = try(each.value.ebs_optimized, var.eks_managed_node_group_defaults.ebs_optimized, null) + key_name = try(each.value.key_name, var.eks_managed_node_group_defaults.key_name, null) + vpc_security_group_ids = compact(concat([try(aws_security_group.node[0].id, "")], try(each.value.vpc_security_group_ids, var.eks_managed_node_group_defaults.vpc_security_group_ids, []))) + launch_template_default_version = try(each.value.launch_template_default_version, var.eks_managed_node_group_defaults.launch_template_default_version, null) + update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.eks_managed_node_group_defaults.update_launch_template_default_version, true) + disable_api_termination = try(each.value.disable_api_termination, var.eks_managed_node_group_defaults.disable_api_termination, null) + kernel_id = try(each.value.kernel_id, var.eks_managed_node_group_defaults.kernel_id, null) + ram_disk_id = try(each.value.ram_disk_id, var.eks_managed_node_group_defaults.ram_disk_id, null) + + block_device_mappings = try(each.value.block_device_mappings, var.eks_managed_node_group_defaults.block_device_mappings, {}) + capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.eks_managed_node_group_defaults.capacity_reservation_specification, null) + cpu_options = try(each.value.cpu_options, var.eks_managed_node_group_defaults.cpu_options, null) + credit_specification = try(each.value.credit_specification, var.eks_managed_node_group_defaults.credit_specification, null) + elastic_gpu_specifications = try(each.value.elastic_gpu_specifications, var.eks_managed_node_group_defaults.elastic_gpu_specifications, null) + elastic_inference_accelerator = try(each.value.elastic_inference_accelerator, var.eks_managed_node_group_defaults.elastic_inference_accelerator, null) + enclave_options = try(each.value.enclave_options, var.eks_managed_node_group_defaults.enclave_options, null) + instance_market_options = try(each.value.instance_market_options, var.eks_managed_node_group_defaults.instance_market_options, null) + license_specifications = try(each.value.license_specifications, var.eks_managed_node_group_defaults.license_specifications, null) + metadata_options = try(each.value.metadata_options, var.eks_managed_node_group_defaults.metadata_options, local.metadata_options) + enable_monitoring = try(each.value.enable_monitoring, var.eks_managed_node_group_defaults.enable_monitoring, true) + network_interfaces = try(each.value.network_interfaces, var.eks_managed_node_group_defaults.network_interfaces, []) + placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, null) + + # IAM role + create_iam_role = try(each.value.create_iam_role, var.eks_managed_node_group_defaults.create_iam_role, true) + iam_role_arn = try(each.value.iam_role_arn, var.eks_managed_node_group_defaults.iam_role_arn, null) + iam_role_name = try(each.value.iam_role_name, var.eks_managed_node_group_defaults.iam_role_name, null) + iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, var.eks_managed_node_group_defaults.iam_role_use_name_prefix, true) + iam_role_path = try(each.value.iam_role_path, var.eks_managed_node_group_defaults.iam_role_path, null) + iam_role_description = try(each.value.iam_role_description, var.eks_managed_node_group_defaults.iam_role_description, "EKS managed node group IAM role") + iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.eks_managed_node_group_defaults.iam_role_permissions_boundary, null) + iam_role_tags = try(each.value.iam_role_tags, var.eks_managed_node_group_defaults.iam_role_tags, {}) + iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.eks_managed_node_group_defaults.iam_role_additional_policies, []) + + # Security group + create_security_group = try(each.value.create_security_group, var.eks_managed_node_group_defaults.create_security_group, true) + security_group_name = try(each.value.security_group_name, var.eks_managed_node_group_defaults.security_group_name, null) + security_group_use_name_prefix = try(each.value.security_group_use_name_prefix, var.eks_managed_node_group_defaults.security_group_use_name_prefix, true) + security_group_description = try(each.value.security_group_description, var.eks_managed_node_group_defaults.security_group_description, "EKS managed node group security group") + vpc_id = try(each.value.vpc_id, var.eks_managed_node_group_defaults.vpc_id, var.vpc_id) + security_group_rules = try(each.value.security_group_rules, var.eks_managed_node_group_defaults.security_group_rules, {}) + security_group_tags = try(each.value.security_group_tags, var.eks_managed_node_group_defaults.security_group_tags, {}) + + tags = merge(var.tags, try(each.value.tags, var.eks_managed_node_group_defaults.tags, {})) +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +module "self_managed_node_group" { + source = "./modules/self-managed-node-group" + + for_each = { for k, v in var.self_managed_node_groups : k => v if var.create } + + cluster_name = aws_eks_cluster.this[0].name + + # Autoscaling Group + name = try(each.value.name, each.key) + use_name_prefix = try(each.value.use_name_prefix, var.self_managed_node_group_defaults.use_name_prefix, true) + + availability_zones = try(each.value.availability_zones, var.self_managed_node_group_defaults.availability_zones, null) + subnet_ids = try(each.value.subnet_ids, var.self_managed_node_group_defaults.subnet_ids, var.subnet_ids) + + min_size = try(each.value.min_size, var.self_managed_node_group_defaults.min_size, 0) + max_size = try(each.value.max_size, var.self_managed_node_group_defaults.max_size, 3) + desired_size = try(each.value.desired_size, var.self_managed_node_group_defaults.desired_size, 1) + capacity_rebalance = try(each.value.capacity_rebalance, var.self_managed_node_group_defaults.capacity_rebalance, null) + min_elb_capacity = try(each.value.min_elb_capacity, var.self_managed_node_group_defaults.min_elb_capacity, null) + wait_for_elb_capacity = try(each.value.wait_for_elb_capacity, var.self_managed_node_group_defaults.wait_for_elb_capacity, null) + wait_for_capacity_timeout = try(each.value.wait_for_capacity_timeout, var.self_managed_node_group_defaults.wait_for_capacity_timeout, null) + default_cooldown = try(each.value.default_cooldown, var.self_managed_node_group_defaults.default_cooldown, null) + protect_from_scale_in = try(each.value.protect_from_scale_in, var.self_managed_node_group_defaults.protect_from_scale_in, null) + + target_group_arns = try(each.value.target_group_arns, var.self_managed_node_group_defaults.target_group_arns, null) + placement_group = try(each.value.placement_group, var.self_managed_node_group_defaults.placement_group, null) + health_check_type = try(each.value.health_check_type, var.self_managed_node_group_defaults.health_check_type, null) + health_check_grace_period = try(each.value.health_check_grace_period, var.self_managed_node_group_defaults.health_check_grace_period, null) + + force_delete = try(each.value.force_delete, var.self_managed_node_group_defaults.force_delete, null) + termination_policies = try(each.value.termination_policies, var.self_managed_node_group_defaults.termination_policies, null) + suspended_processes = try(each.value.suspended_processes, var.self_managed_node_group_defaults.suspended_processes, null) + max_instance_lifetime = try(each.value.max_instance_lifetime, var.self_managed_node_group_defaults.max_instance_lifetime, null) + + enabled_metrics = try(each.value.enabled_metrics, var.self_managed_node_group_defaults.enabled_metrics, null) + metrics_granularity = try(each.value.metrics_granularity, var.self_managed_node_group_defaults.metrics_granularity, null) + service_linked_role_arn = try(each.value.service_linked_role_arn, var.self_managed_node_group_defaults.service_linked_role_arn, null) + + initial_lifecycle_hooks = try(each.value.initial_lifecycle_hooks, var.self_managed_node_group_defaults.initial_lifecycle_hooks, []) + instance_refresh = try(each.value.instance_refresh, var.self_managed_node_group_defaults.instance_refresh, null) + use_mixed_instances_policy = try(each.value.use_mixed_instances_policy, var.self_managed_node_group_defaults.use_mixed_instances_policy, false) + warm_pool = try(each.value.warm_pool, var.self_managed_node_group_defaults.warm_pool, null) + + create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, false) + schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, null) + + delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) + + # User data + platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, "linux") + cluster_endpoint = try(aws_eks_cluster.this[0].endpoint, "") + cluster_auth_base64 = try(aws_eks_cluster.this[0].certificate_authority[0].data, "") + pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.self_managed_node_group_defaults.pre_bootstrap_user_data, "") + post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "") + bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "") + user_data_template_path = try(each.value.user_data_template_path, var.self_managed_node_group_defaults.user_data_template_path, "") + + # Launch Template + create_launch_template = try(each.value.create_launch_template, var.self_managed_node_group_defaults.create_launch_template, true) + launch_template_name = try(each.value.launch_template_name, each.key) + launch_template_version = try(each.value.launch_template_version, var.self_managed_node_group_defaults.launch_template_version, null) + launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") - create_eks = var.create_eks + ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) + ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") + cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, var.cluster_version) + instance_type = try(each.value.instance_type, var.self_managed_node_group_defaults.instance_type, "m6i.large") + key_name = try(each.value.key_name, var.self_managed_node_group_defaults.key_name, null) - cluster_name = local.cluster_name - cluster_endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 + vpc_security_group_ids = compact(concat([try(aws_security_group.node[0].id, "")], try(each.value.vpc_security_group_ids, var.self_managed_node_group_defaults.vpc_security_group_ids, []))) + cluster_security_group_id = local.cluster_security_group_id + launch_template_default_version = try(each.value.launch_template_default_version, var.self_managed_node_group_defaults.launch_template_default_version, null) + update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.self_managed_node_group_defaults.update_launch_template_default_version, true) + disable_api_termination = try(each.value.disable_api_termination, var.self_managed_node_group_defaults.disable_api_termination, null) + instance_initiated_shutdown_behavior = try(each.value.instance_initiated_shutdown_behavior, var.self_managed_node_group_defaults.instance_initiated_shutdown_behavior, null) + kernel_id = try(each.value.kernel_id, var.self_managed_node_group_defaults.kernel_id, null) + ram_disk_id = try(each.value.ram_disk_id, var.self_managed_node_group_defaults.ram_disk_id, null) - default_iam_role_arn = coalescelist(aws_iam_role.workers[*].arn, [""])[0] - ebs_optimized_not_supported = local.ebs_optimized_not_supported - workers_group_defaults = local.workers_group_defaults - worker_security_group_id = local.worker_security_group_id - worker_additional_security_group_ids = var.worker_additional_security_group_ids + block_device_mappings = try(each.value.block_device_mappings, var.self_managed_node_group_defaults.block_device_mappings, []) + capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.self_managed_node_group_defaults.capacity_reservation_specification, null) + cpu_options = try(each.value.cpu_options, var.self_managed_node_group_defaults.cpu_options, null) + credit_specification = try(each.value.credit_specification, var.self_managed_node_group_defaults.credit_specification, null) + elastic_gpu_specifications = try(each.value.elastic_gpu_specifications, var.self_managed_node_group_defaults.elastic_gpu_specifications, null) + elastic_inference_accelerator = try(each.value.elastic_inference_accelerator, var.self_managed_node_group_defaults.elastic_inference_accelerator, null) + enclave_options = try(each.value.enclave_options, var.self_managed_node_group_defaults.enclave_options, null) + hibernation_options = try(each.value.hibernation_options, var.self_managed_node_group_defaults.hibernation_options, null) + instance_market_options = try(each.value.instance_market_options, var.self_managed_node_group_defaults.instance_market_options, null) + license_specifications = try(each.value.license_specifications, var.self_managed_node_group_defaults.license_specifications, null) + metadata_options = try(each.value.metadata_options, var.self_managed_node_group_defaults.metadata_options, local.metadata_options) + enable_monitoring = try(each.value.enable_monitoring, var.self_managed_node_group_defaults.enable_monitoring, true) + network_interfaces = try(each.value.network_interfaces, var.self_managed_node_group_defaults.network_interfaces, []) + placement = try(each.value.placement, var.self_managed_node_group_defaults.placement, null) - node_groups_defaults = var.node_groups_defaults - node_groups = var.node_groups + # IAM role + create_iam_instance_profile = try(each.value.create_iam_instance_profile, var.self_managed_node_group_defaults.create_iam_instance_profile, true) + iam_instance_profile_arn = try(each.value.iam_instance_profile_arn, var.self_managed_node_group_defaults.iam_instance_profile_arn, null) + iam_role_name = try(each.value.iam_role_name, var.self_managed_node_group_defaults.iam_role_name, null) + iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, var.self_managed_node_group_defaults.iam_role_use_name_prefix, true) + iam_role_path = try(each.value.iam_role_path, var.self_managed_node_group_defaults.iam_role_path, null) + iam_role_description = try(each.value.iam_role_description, var.self_managed_node_group_defaults.iam_role_description, "Self managed node group IAM role") + iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.self_managed_node_group_defaults.iam_role_permissions_boundary, null) + iam_role_tags = try(each.value.iam_role_tags, var.self_managed_node_group_defaults.iam_role_tags, {}) + iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.self_managed_node_group_defaults.iam_role_attach_cni_policy, true) + iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.self_managed_node_group_defaults.iam_role_additional_policies, []) - tags = var.tags + # Security group + create_security_group = try(each.value.create_security_group, var.self_managed_node_group_defaults.create_security_group, true) + security_group_name = try(each.value.security_group_name, var.self_managed_node_group_defaults.security_group_name, null) + security_group_use_name_prefix = try(each.value.security_group_use_name_prefix, var.self_managed_node_group_defaults.security_group_use_name_prefix, true) + security_group_description = try(each.value.security_group_description, var.self_managed_node_group_defaults.security_group_description, "Self managed node group security group") + vpc_id = try(each.value.vpc_id, var.self_managed_node_group_defaults.vpc_id, var.vpc_id) + security_group_rules = try(each.value.security_group_rules, var.self_managed_node_group_defaults.security_group_rules, {}) + security_group_tags = try(each.value.security_group_tags, var.self_managed_node_group_defaults.security_group_tags, {}) - depends_on = [ - aws_eks_cluster.this, - aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy, - aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy, - aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly - ] + tags = merge(var.tags, try(each.value.tags, var.self_managed_node_group_defaults.tags, {})) + propagate_tags = try(each.value.propagate_tags, var.self_managed_node_group_defaults.propagate_tags, []) } diff --git a/outputs.tf b/outputs.tf index 8676285368..72ba73d676 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,211 +1,174 @@ -output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready." - value = local.cluster_id - - # So that calling plans wait for the cluster to be available before attempting to use it. - # There is no need to duplicate this datasource - depends_on = [data.http.wait_for_cluster] -} +################################################################################ +# Cluster +################################################################################ output "cluster_arn" { - description = "The Amazon Resource Name (ARN) of the cluster." - value = local.cluster_arn + description = "The Amazon Resource Name (ARN) of the cluster" + value = try(aws_eks_cluster.this[0].arn, "") } output "cluster_certificate_authority_data" { - description = "Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster." - value = local.cluster_auth_base64 + description = "Base64 encoded certificate data required to communicate with the cluster" + value = try(aws_eks_cluster.this[0].certificate_authority[0].data, "") } output "cluster_endpoint" { - description = "The endpoint for your EKS Kubernetes API." - value = local.cluster_endpoint + description = "Endpoint for your Kubernetes API server" + value = try(aws_eks_cluster.this[0].endpoint, "") } -output "cluster_version" { - description = "The Kubernetes server version for the EKS cluster." - value = element(concat(aws_eks_cluster.this[*].version, [""]), 0) -} - -output "cluster_security_group_id" { - description = "Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console." - value = local.cluster_security_group_id +output "cluster_id" { + description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = try(aws_eks_cluster.this[0].id, "") } -output "config_map_aws_auth" { - description = "A kubernetes configuration to authenticate to this EKS cluster." - value = kubernetes_config_map.aws_auth.* +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "") } -output "cluster_iam_role_name" { - description = "IAM role name of the EKS cluster." - value = local.cluster_iam_role_name +output "cluster_platform_version" { + description = "Platform version for the cluster" + value = try(aws_eks_cluster.this[0].platform_version, "") } -output "cluster_iam_role_arn" { - description = "IAM role ARN of the EKS cluster." - value = local.cluster_iam_role_arn -} - -output "cluster_oidc_issuer_url" { - description = "The URL on the EKS cluster OIDC Issuer" - value = local.cluster_oidc_issuer_url +output "cluster_status" { + description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" + value = try(aws_eks_cluster.this[0].status, "") } output "cluster_primary_security_group_id" { - description = "The cluster primary security group ID created by the EKS cluster on 1.14 or later. Referred to as 'Cluster security group' in the EKS console." - value = local.cluster_primary_security_group_id + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = try(aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id, "") } -output "cloudwatch_log_group_name" { - description = "Name of cloudwatch log group created" - value = element(concat(aws_cloudwatch_log_group.this[*].name, [""]), 0) +################################################################################ +# Cluster Security Group +################################################################################ + +output "cluster_security_group_arn" { + description = "Amazon Resource Name (ARN) of the cluster security group" + value = try(aws_security_group.cluster[0].arn, "") } -output "cloudwatch_log_group_arn" { - description = "Arn of cloudwatch log group created" - value = element(concat(aws_cloudwatch_log_group.this[*].arn, [""]), 0) +output "cluster_security_group_id" { + description = "ID of the cluster security group" + value = try(aws_security_group.cluster[0].id, "") } -output "kubeconfig" { - description = "kubectl config file contents for this EKS cluster. Will block on cluster creation until the cluster is really ready." - value = local.kubeconfig +################################################################################ +# Node Security Group +################################################################################ - # So that calling plans wait for the cluster to be available before attempting to use it. - # There is no need to duplicate this datasource - depends_on = [data.http.wait_for_cluster] +output "node_security_group_arn" { + description = "Amazon Resource Name (ARN) of the node shared security group" + value = try(aws_security_group.node[0].arn, "") } -output "kubeconfig_filename" { - description = "The filename of the generated kubectl config. Will block on cluster creation until the cluster is really ready." - value = concat(local_file.kubeconfig.*.filename, [""])[0] - - # So that calling plans wait for the cluster to be available before attempting to use it. - # There is no need to duplicate this datasource - depends_on = [data.http.wait_for_cluster] +output "node_security_group_id" { + description = "ID of the node shared security group" + value = try(aws_security_group.node[0].id, "") } +################################################################################ +# IRSA +################################################################################ + output "oidc_provider_arn" { - description = "The ARN of the OIDC Provider if `enable_irsa = true`." - value = var.enable_irsa ? concat(aws_iam_openid_connect_provider.oidc_provider[*].arn, [""])[0] : null + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + value = try(aws_iam_openid_connect_provider.oidc_provider[0].arn, "") } -output "workers_asg_arns" { - description = "IDs of the autoscaling groups containing workers." - value = concat( - aws_autoscaling_group.workers.*.arn, - aws_autoscaling_group.workers_launch_template.*.arn, - ) -} +################################################################################ +# IAM Role +################################################################################ -output "workers_asg_names" { - description = "Names of the autoscaling groups containing workers." - value = concat( - aws_autoscaling_group.workers.*.id, - aws_autoscaling_group.workers_launch_template.*.id, - ) +output "cluster_iam_role_name" { + description = "IAM role name of the EKS cluster" + value = try(aws_iam_role.this[0].name, "") } -output "workers_user_data" { - description = "User data of worker groups" - value = concat( - local.launch_configuration_userdata_rendered, - local.launch_template_userdata_rendered, - ) +output "cluster_iam_role_arn" { + description = "IAM role ARN of the EKS cluster" + value = try(aws_iam_role.this[0].arn, "") } -output "workers_default_ami_id" { - description = "ID of the default worker group AMI" - value = local.default_ami_id_linux +output "cluster_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, "") } -output "workers_default_ami_id_windows" { - description = "ID of the default Windows worker group AMI" - value = local.default_ami_id_windows -} +################################################################################ +# EKS Addons +################################################################################ -output "workers_launch_template_ids" { - description = "IDs of the worker launch templates." - value = aws_launch_template.workers_launch_template.*.id +output "cluster_addons" { + description = "Map of attribute maps for all EKS cluster addons enabled" + value = aws_eks_addon.this } -output "workers_launch_template_arns" { - description = "ARNs of the worker launch templates." - value = aws_launch_template.workers_launch_template.*.arn -} +################################################################################ +# EKS Identity Provider +################################################################################ -output "workers_launch_template_latest_versions" { - description = "Latest versions of the worker launch templates." - value = aws_launch_template.workers_launch_template.*.latest_version +output "cluster_identity_providers" { + description = "Map of attribute maps for all EKS identity providers enabled" + value = aws_eks_identity_provider_config.this } -output "worker_security_group_id" { - description = "Security group ID attached to the EKS workers." - value = local.worker_security_group_id -} +################################################################################ +# CloudWatch Log Group +################################################################################ -output "worker_iam_instance_profile_arns" { - description = "default IAM instance profile ARN for EKS worker groups" - value = concat( - aws_iam_instance_profile.workers.*.arn, - aws_iam_instance_profile.workers_launch_template.*.arn - ) +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = try(aws_cloudwatch_log_group.this[0].name, "") } -output "worker_iam_instance_profile_names" { - description = "default IAM instance profile name for EKS worker groups" - value = concat( - aws_iam_instance_profile.workers.*.name, - aws_iam_instance_profile.workers_launch_template.*.name - ) +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = try(aws_cloudwatch_log_group.this[0].arn, "") } -output "worker_iam_role_name" { - description = "default IAM role name for EKS worker groups" - value = coalescelist( - aws_iam_role.workers.*.name, - data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, - data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, - [""] - )[0] -} +################################################################################ +# Fargate Profile +################################################################################ -output "worker_iam_role_arn" { - description = "default IAM role ARN for EKS worker groups" - value = coalescelist( - aws_iam_role.workers.*.arn, - data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_arn, - data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_arn, - [""] - )[0] +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.fargate_profile } -output "fargate_profile_ids" { - description = "EKS Cluster name and EKS Fargate Profile names separated by a colon (:)." - value = module.fargate.fargate_profile_ids -} +################################################################################ +# EKS Managed Node Group +################################################################################ -output "fargate_profile_arns" { - description = "Amazon Resource Name (ARN) of the EKS Fargate Profiles." - value = module.fargate.fargate_profile_arns +output "eks_managed_node_groups" { + description = "Map of attribute maps for all EKS managed node groups created" + value = module.eks_managed_node_group } -output "fargate_iam_role_name" { - description = "IAM role name for EKS Fargate pods" - value = module.fargate.iam_role_name -} +################################################################################ +# Self Managed Node Group +################################################################################ -output "fargate_iam_role_arn" { - description = "IAM role ARN for EKS Fargate pods" - value = module.fargate.iam_role_arn +output "self_managed_node_groups" { + description = "Map of attribute maps for all self managed node groups created" + value = module.self_managed_node_group } -output "node_groups" { - description = "Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys" - value = module.node_groups.node_groups -} +################################################################################ +# Additional +################################################################################ -output "security_group_rule_cluster_https_worker_ingress" { - description = "Security group rule responsible for allowing pods to communicate with the EKS cluster API." - value = aws_security_group_rule.cluster_https_worker_ingress +output "aws_auth_configmap_yaml" { + description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" + value = templatefile("${path.module}/templates/aws_auth_cm.tpl", + { + eks_managed_role_arns = [for group in module.eks_managed_node_group : group.iam_role_arn] + self_managed_role_arns = [for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"] + win32_self_managed_role_arns = [for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"] + fargate_profile_arns = [for group in module.fargate_profile : group.fargate_profile_arn] + } + ) } diff --git a/templates/aws_auth_cm.tpl b/templates/aws_auth_cm.tpl new file mode 100644 index 0000000000..abf2102a54 --- /dev/null +++ b/templates/aws_auth_cm.tpl @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aws-auth + namespace: kube-system +data: + mapRoles: | +%{ for role in eks_managed_role_arns ~} + - rolearn: ${role} + username: system:node:{{EC2PrivateDNSName}} + groups: + - system:bootstrappers + - system:nodes +%{ endfor ~} +%{ for role in self_managed_role_arns ~} + - rolearn: ${role} + username: system:node:{{EC2PrivateDNSName}} + groups: + - system:bootstrappers + - system:nodes +%{ endfor ~} +%{ for role in win32_self_managed_role_arns ~} + - rolearn: ${role} + username: system:node:{{EC2PrivateDNSName}} + groups: + - eks:kube-proxy-windows + - system:bootstrappers + - system:nodes +%{ endfor ~} +%{ for role in fargate_profile_arns ~} + - rolearn: ${role} + username: system:node:{{SessionName}} + groups: + - system:bootstrappers + - system:nodes + - system:node-proxier +%{ endfor ~} diff --git a/templates/bottlerocket_user_data.tpl b/templates/bottlerocket_user_data.tpl new file mode 100644 index 0000000000..640c801438 --- /dev/null +++ b/templates/bottlerocket_user_data.tpl @@ -0,0 +1,7 @@ +%{ if enable_bootstrap_user_data ~} +[settings.kubernetes] +"cluster-name" = "${cluster_name}" +"api-server" = "${cluster_endpoint}" +"cluster-certificate" = "${cluster_auth_base64}" +%{ endif ~} +${bootstrap_extra_args ~} diff --git a/templates/kubeconfig.tpl b/templates/kubeconfig.tpl deleted file mode 100644 index 5004243bec..0000000000 --- a/templates/kubeconfig.tpl +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: v1 -preferences: {} -kind: Config - -clusters: -- cluster: - server: ${endpoint} - certificate-authority-data: ${cluster_auth_base64} - name: ${kubeconfig_name} - -contexts: -- context: - cluster: ${kubeconfig_name} - user: ${kubeconfig_name} - name: ${kubeconfig_name} - -current-context: ${kubeconfig_name} - -users: -- name: ${kubeconfig_name} - user: - exec: - apiVersion: ${aws_authenticator_kubeconfig_apiversion} - command: ${aws_authenticator_command} - args: -%{~ for i in aws_authenticator_command_args } - - "${i}" -%{~ endfor ~} -%{ for i in aws_authenticator_additional_args } - - ${i} -%{~ endfor ~} -%{ if length(aws_authenticator_env_variables) > 0 } - env: - %{~ for k, v in aws_authenticator_env_variables ~} - - name: ${k} - value: ${v} - %{~ endfor ~} -%{ endif } diff --git a/templates/linux_user_data.tpl b/templates/linux_user_data.tpl new file mode 100644 index 0000000000..14acbd2aff --- /dev/null +++ b/templates/linux_user_data.tpl @@ -0,0 +1,14 @@ +%{ if enable_bootstrap_user_data ~} +#!/bin/bash +set -e +%{ endif ~} +${pre_bootstrap_user_data ~} +%{ if length(cluster_service_ipv4_cidr) > 0 ~} +export SERVICE_IPV4_CIDR=${cluster_service_ipv4_cidr} +%{ endif ~} +%{ if enable_bootstrap_user_data ~} +B64_CLUSTER_CA=${cluster_auth_base64} +API_SERVER_URL=${cluster_endpoint} +/etc/eks/bootstrap.sh ${cluster_name} ${bootstrap_extra_args} --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +${post_bootstrap_user_data ~} +%{ endif ~} diff --git a/templates/userdata.sh.tpl b/templates/userdata.sh.tpl deleted file mode 100644 index cf314b8800..0000000000 --- a/templates/userdata.sh.tpl +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -# Allow user supplied pre userdata code -${pre_userdata} - -# Bootstrap and join the cluster -/etc/eks/bootstrap.sh --b64-cluster-ca '${cluster_auth_base64}' --apiserver-endpoint '${endpoint}' ${bootstrap_extra_args} --kubelet-extra-args "${kubelet_extra_args}" '${cluster_name}' - -# Allow user supplied userdata code -${additional_userdata} diff --git a/templates/userdata_windows.tpl b/templates/windows_user_data.tpl similarity index 53% rename from templates/userdata_windows.tpl rename to templates/windows_user_data.tpl index e8856838f1..5000850604 100644 --- a/templates/userdata_windows.tpl +++ b/templates/windows_user_data.tpl @@ -1,11 +1,9 @@ -${pre_userdata} - +${pre_bootstrap_user_data ~} [string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" [string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' [string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" -& $EKSBootstrapScriptFile -EKSClusterName ${cluster_name} -KubeletExtraArgs "${kubelet_extra_args}" 3>&1 4>&1 5>&1 6>&1 +& $EKSBootstrapScriptFile -EKSClusterName ${cluster_name} -APIServerEndpoint ${cluster_endpoint} -Base64ClusterCA ${cluster_auth_base64} ${bootstrap_extra_args} 3>&1 4>&1 5>&1 6>&1 $LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } - -${additional_userdata} +${post_bootstrap_user_data ~} diff --git a/variables.tf b/variables.tf index 575c76a33c..12ff69ee33 100644 --- a/variables.tf +++ b/variables.tf @@ -1,436 +1,352 @@ -variable "cluster_enabled_log_types" { - description = "A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html)" - type = list(string) - default = [] +variable "create" { + description = "Controls if EKS resources should be created (affects nearly all resources)" + type = bool + default = true } -variable "cluster_log_kms_key_id" { - description = "If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html)" - type = string - default = "" +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} } -variable "cluster_log_retention_in_days" { - description = "Number of days to retain log events. Default retention - 90 days." - type = number - default = 90 -} +################################################################################ +# Cluster +################################################################################ variable "cluster_name" { - description = "Name of the EKS cluster. Also used as a prefix in names of related resources." - type = string - default = "" -} - -variable "cluster_security_group_id" { - description = "If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the workers" + description = "Name of the EKS cluster" type = string default = "" } variable "cluster_version" { - description = "Kubernetes minor version to use for the EKS cluster (for example 1.21)." + description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.21`)" type = string default = null } -variable "kubeconfig_output_path" { - description = "Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`." - type = string - default = "./" +variable "cluster_enabled_log_types" { + description = "A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html)" + type = list(string) + default = ["audit", "api", "authenticator"] } -variable "kubeconfig_file_permission" { - description = "File permission of the Kubectl config file containing cluster configuration saved to `kubeconfig_output_path.`" - type = string - default = "0600" +variable "cluster_additional_security_group_ids" { + description = "List of additional, externally created security group IDs to attach to the cluster control plane" + type = list(string) + default = [] } -variable "write_kubeconfig" { - description = "Whether to write a Kubectl config file containing the cluster configuration. Saved to `kubeconfig_output_path`." - type = bool - default = true +variable "subnet_ids" { + description = "A list of subnet IDs where the EKS cluster (ENIs) will be provisioned along with the nodes/node groups. Node groups can be deployed within a different set of subnet IDs from within the node group configuration" + type = list(string) + default = [] } -variable "default_platform" { - description = "Default platform name. Valid options are `linux` and `windows`." - type = string - default = "linux" +variable "cluster_endpoint_private_access" { + description = "Indicates whether or not the Amazon EKS private API server endpoint is enabled" + type = bool + default = false } -variable "manage_aws_auth" { - description = "Whether to apply the aws-auth configmap file." +variable "cluster_endpoint_public_access" { + description = "Indicates whether or not the Amazon EKS public API server endpoint is enabled" type = bool default = true } -variable "aws_auth_additional_labels" { - description = "Additional kubernetes labels applied on aws-auth ConfigMap" - default = {} - type = map(string) -} - -variable "map_accounts" { - description = "Additional AWS account numbers to add to the aws-auth configmap." +variable "cluster_endpoint_public_access_cidrs" { + description = "List of CIDR blocks which can access the Amazon EKS public API server endpoint" type = list(string) - default = [] + default = ["0.0.0.0/0"] } -variable "map_roles" { - description = "Additional IAM roles to add to the aws-auth configmap." - type = list(object({ - rolearn = string - username = string - groups = list(string) - })) - default = [] +variable "cluster_service_ipv4_cidr" { + description = "The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" + type = string + default = null } -variable "map_users" { - description = "Additional IAM users to add to the aws-auth configmap." +variable "cluster_encryption_config" { + description = "Configuration block with encryption configuration for the cluster" type = list(object({ - userarn = string - username = string - groups = list(string) + provider_key_arn = string + resources = list(string) })) default = [] } -variable "fargate_subnets" { - description = "A list of subnets to place fargate workers within (if different from subnets)." - type = list(string) - default = [] -} - -variable "subnets" { - description = "A list of subnets to place the EKS cluster and workers within." - type = list(string) - default = [] -} - -variable "tags" { - description = "A map of tags to add to all resources. Tags added to launch configuration or templates override these values for ASG Tags only." +variable "cluster_tags" { + description = "A map of additional tags to add to the cluster" type = map(string) default = {} } -variable "cluster_tags" { - description = "A map of tags to add to just the eks resource." +variable "cluster_timeouts" { + description = "Create, update, and delete timeout configurations for the cluster" type = map(string) default = {} } -variable "vpc_id" { - description = "VPC where the cluster and workers will be deployed." - type = string - default = null -} - -variable "worker_groups" { - description = "A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys." - type = any - default = [] -} +################################################################################ +# CloudWatch Log Group +################################################################################ -variable "workers_group_defaults" { - description = "Override default values for target groups. See workers_group_defaults_defaults in local.tf for valid keys." - type = any - default = {} +variable "create_cloudwatch_log_group" { + description = "Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled" + type = bool + default = true } -variable "worker_groups_launch_template" { - description = "A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys." - type = any - default = [] +variable "cloudwatch_log_group_retention_in_days" { + description = "Number of days to retain log events. Default retention - 90 days" + type = number + default = 90 } -variable "worker_security_group_id" { - description = "If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster." +variable "cloudwatch_log_group_kms_key_id" { + description = "If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html)" type = string - default = "" + default = null } -variable "worker_ami_name_filter" { - description = "Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used." - type = string - default = "" +################################################################################ +# Cluster Security Group +################################################################################ + +variable "create_cluster_security_group" { + description = "Determines if a security group is created for the cluster or use the existing `cluster_security_group_id`" + type = bool + default = true } -variable "worker_ami_name_filter_windows" { - description = "Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used." +variable "cluster_security_group_id" { + description = "Existing security group ID to be attached to the cluster. Required if `create_cluster_security_group` = `false`" type = string default = "" } -variable "worker_ami_owner_id" { - description = "The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft')." +variable "vpc_id" { + description = "ID of the VPC where the cluster and its nodes will be provisioned" type = string - default = "amazon" + default = null } -variable "worker_ami_owner_id_windows" { - description = "The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft')." +variable "cluster_security_group_name" { + description = "Name to use on cluster security group created" type = string - default = "amazon" -} - -variable "worker_additional_security_group_ids" { - description = "A list of additional security group ids to attach to worker instances" - type = list(string) - default = [] -} - -variable "worker_sg_ingress_from_port" { - description = "Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443)." - type = number - default = 1025 -} - -variable "workers_additional_policies" { - description = "Additional policies to be added to workers" - type = list(string) - default = [] + default = null } -variable "kubeconfig_api_version" { - description = "KubeConfig API version. Defaults to client.authentication.k8s.io/v1alpha1" - type = string - default = "client.authentication.k8s.io/v1alpha1" -} -variable "kubeconfig_aws_authenticator_command" { - description = "Command to use to fetch AWS EKS credentials." +variable "cluster_security_group_use_name_prefix" { + description = "Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix" type = string - default = "aws-iam-authenticator" + default = true } -variable "kubeconfig_aws_authenticator_command_args" { - description = "Default arguments passed to the authenticator command. Defaults to [token -i $cluster_name]." - type = list(string) - default = [] +variable "cluster_security_group_description" { + description = "Description of the cluster security group created" + type = string + default = "EKS cluster security group" } -variable "kubeconfig_aws_authenticator_additional_args" { - description = "Any additional arguments to pass to the authenticator such as the role to assume. e.g. [\"-r\", \"MyEksRole\"]." - type = list(string) - default = [] +variable "cluster_security_group_additional_rules" { + description = "List of additional security group rules to add to the cluster security group created" + type = map(any) + default = {} } -variable "kubeconfig_aws_authenticator_env_variables" { - description = "Environment variables that should be used when executing the authenticator. e.g. { AWS_PROFILE = \"eks\"}." +variable "cluster_security_group_tags" { + description = "A map of additional tags to add to the cluster security group created" type = map(string) default = {} } -variable "kubeconfig_name" { - description = "Override the default name used for items kubeconfig." - type = string - default = "" -} +################################################################################ +# Node Security Group +################################################################################ -variable "cluster_create_timeout" { - description = "Timeout value when creating the EKS cluster." - type = string - default = "30m" +variable "create_node_security_group" { + description = "Determines whether to create a security group for the node groups or use the existing `node_security_group_id`" + type = bool + default = true } -variable "cluster_delete_timeout" { - description = "Timeout value when deleting the EKS cluster." +variable "node_security_group_id" { + description = "ID of an existing security group to attach to the node groups created" type = string - default = "15m" + default = "" } -variable "cluster_update_timeout" { - description = "Timeout value when updating the EKS cluster." +variable "node_security_group_name" { + description = "Name to use on node security group created" type = string - default = "60m" -} - -variable "cluster_create_security_group" { - description = "Whether to create a security group for the cluster or attach the cluster to `cluster_security_group_id`." - type = bool - default = true + default = null } -variable "worker_create_security_group" { - description = "Whether to create a security group for the workers or attach the workers to `worker_security_group_id`." - type = bool +variable "node_security_group_use_name_prefix" { + description = "Determines whether node security group name (`node_security_group_name`) is used as a prefix" + type = string default = true } -variable "worker_create_initial_lifecycle_hooks" { - description = "Whether to create initial lifecycle hooks provided in worker groups." - type = bool - default = false +variable "node_security_group_description" { + description = "Description of the node security group created" + type = string + default = "EKS node shared security group" } -variable "worker_create_cluster_primary_security_group_rules" { - description = "Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group." - type = bool - default = false +variable "node_security_group_additional_rules" { + description = "List of additional security group rules to add to the node security group created" + type = map(any) + default = {} } -variable "permissions_boundary" { - description = "If provided, all IAM roles will be created with this permissions boundary attached." - type = string - default = null +variable "node_security_group_tags" { + description = "A map of additional tags to add to the node security group created" + type = map(string) + default = {} } -variable "iam_path" { - description = "If provided, all IAM roles will be created on this path." - type = string - default = "/" -} +################################################################################ +# IRSA +################################################################################ -variable "cluster_create_endpoint_private_access_sg_rule" { - description = "Whether to create security group rules for the access to the Amazon EKS private API server endpoint. When is `true`, `cluster_endpoint_private_access_cidrs` must be setted." +variable "enable_irsa" { + description = "Determines whether to create an OpenID Connect Provider for EKS to enable IRSA" type = bool default = false } -variable "cluster_endpoint_private_access_cidrs" { - description = "List of CIDR blocks which can access the Amazon EKS private API server endpoint. To use this `cluster_endpoint_private_access` and `cluster_create_endpoint_private_access_sg_rule` must be set to `true`." - type = list(string) - default = null -} - -variable "cluster_endpoint_private_access_sg" { - description = "List of security group IDs which can access the Amazon EKS private API server endpoint. To use this `cluster_endpoint_private_access` and `cluster_create_endpoint_private_access_sg_rule` must be set to `true`." +variable "openid_connect_audiences" { + description = "List of OpenID Connect audience client IDs to add to the IRSA provider" type = list(string) - default = null + default = [] } -variable "cluster_endpoint_private_access" { - description = "Indicates whether or not the Amazon EKS private API server endpoint is enabled." - type = bool - default = false -} +################################################################################ +# Cluster IAM Role +################################################################################ -variable "cluster_endpoint_public_access" { - description = "Indicates whether or not the Amazon EKS public API server endpoint is enabled. When it's set to `false` ensure to have a proper private access with `cluster_endpoint_private_access = true`." +variable "create_iam_role" { + description = "Determines whether a an IAM role is created or to use an existing IAM role" type = bool default = true } -variable "cluster_endpoint_public_access_cidrs" { - description = "List of CIDR blocks which can access the Amazon EKS public API server endpoint." - type = list(string) - default = ["0.0.0.0/0"] +variable "iam_role_arn" { + description = "Existing IAM role ARN for the cluster. Required if `create_iam_role` is set to `false`" + type = string + default = null } -variable "manage_cluster_iam_resources" { - description = "Whether to let the module manage cluster IAM resources. If set to false, cluster_iam_role_name must be specified." - type = bool - default = true +variable "iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null } -variable "cluster_iam_role_name" { - description = "IAM role name for the cluster. If manage_cluster_iam_resources is set to false, set this to reuse an existing IAM role. If manage_cluster_iam_resources is set to true, set this to force the created role name." +variable "iam_role_use_name_prefix" { + description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" type = string - default = "" + default = true } -variable "manage_worker_iam_resources" { - description = "Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers." - type = bool - default = true +variable "iam_role_path" { + description = "Cluster IAM role path" + type = string + default = null } -variable "workers_role_name" { - description = "User defined workers role name." +variable "iam_role_description" { + description = "Description of the role" type = string - default = "" + default = null } -variable "attach_worker_cni_policy" { - description = "Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the default worker IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster." - type = bool - default = true +variable "iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null } -variable "create_eks" { - description = "Controls if EKS resources should be created (it affects almost all resources)" - type = bool - default = true +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = list(string) + default = [] } -variable "node_groups_defaults" { - description = "Map of values to be applied to all node groups. See `node_groups` module's documentation for more details" - type = any +variable "iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) default = {} } -variable "node_groups" { - description = "Map of map of node groups to create. See `node_groups` module's documentation for more details" +################################################################################ +# EKS Addons +################################################################################ + +variable "cluster_addons" { + description = "Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name`" type = any default = {} } -variable "enable_irsa" { - description = "Whether to create OpenID Connect Provider for EKS to enable IRSA" - type = bool - default = false -} +################################################################################ +# EKS Identity Provider +################################################################################ -variable "eks_oidc_root_ca_thumbprint" { - type = string - description = "Thumbprint of Root CA for EKS OIDC, Valid until 2037" - default = "9e99a48a9960b14926bb7f3b02e22da2b0ab7280" +variable "cluster_identity_providers" { + description = "Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA" + type = any + default = {} } -variable "cluster_encryption_config" { - description = "Configuration block with encryption configuration for the cluster. See examples/secrets_encryption/main.tf for example format" - type = list(object({ - provider_key_arn = string - resources = list(string) - })) - default = [] -} +################################################################################ +# Fargate +################################################################################ variable "fargate_profiles" { - description = "Fargate profiles to create. See `fargate_profile` keys section in fargate submodule's README.md for more details" + description = "Map of Fargate Profile definitions to create" type = any default = {} } -variable "create_fargate_pod_execution_role" { - description = "Controls if the EKS Fargate pod execution IAM role should be created." - type = bool - default = true +variable "fargate_profile_defaults" { + description = "Map of Fargate Profile default configurations" + type = any + default = {} } -variable "fargate_pod_execution_role_name" { - description = "The IAM Role that provides permissions for the EKS Fargate Profile." - type = string - default = null -} +################################################################################ +# Self Managed Node Group +################################################################################ -variable "cluster_service_ipv4_cidr" { - description = "service ipv4 cidr for the kubernetes cluster" - type = string - default = null +variable "self_managed_node_groups" { + description = "Map of self-managed node group definitions to create" + type = any + default = {} } -variable "cluster_egress_cidrs" { - description = "List of CIDR blocks that are permitted for cluster egress traffic." - type = list(string) - default = ["0.0.0.0/0"] +variable "self_managed_node_group_defaults" { + description = "Map of self-managed node group default configurations" + type = any + default = {} } -variable "workers_egress_cidrs" { - description = "List of CIDR blocks that are permitted for workers egress traffic." - type = list(string) - default = ["0.0.0.0/0"] -} +################################################################################ +# EKS Managed Node Group +################################################################################ -variable "wait_for_cluster_timeout" { - description = "A timeout (in seconds) to wait for cluster to be available." - type = number - default = 300 +variable "eks_managed_node_groups" { + description = "Map of EKS managed node group definitions to create" + type = any + default = {} } -variable "openid_connect_audiences" { - description = "List of OpenID Connect audience client IDs to add to the IRSA provider." - type = list(string) - default = [] +variable "eks_managed_node_group_defaults" { + description = "Map of EKS managed node group default configurations" + type = any + default = {} } - diff --git a/versions.tf b/versions.tf index 602d086f47..1969855a84 100644 --- a/versions.tf +++ b/versions.tf @@ -4,23 +4,11 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.56" + version = ">= 3.64" } - local = { - source = "hashicorp/local" - version = ">= 1.4" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 1.11.1" - } - cloudinit = { - source = "hashicorp/cloudinit" - version = ">= 2.0" - } - http = { - source = "terraform-aws-modules/http" - version = ">= 2.4.1" + tls = { + source = "hashicorp/tls" + version = ">= 2.2" } } } diff --git a/workers.tf b/workers.tf deleted file mode 100644 index 610c41709e..0000000000 --- a/workers.tf +++ /dev/null @@ -1,533 +0,0 @@ -# Worker Groups using Launch Configurations - -resource "aws_autoscaling_group" "workers" { - count = var.create_eks ? local.worker_group_launch_configuration_count : 0 - - name_prefix = join( - "-", - compact( - [ - local.cluster_name, - lookup(var.worker_groups[count.index], "name", count.index) - ] - ) - ) - desired_capacity = lookup( - var.worker_groups[count.index], - "asg_desired_capacity", - local.workers_group_defaults["asg_desired_capacity"], - ) - max_size = lookup( - var.worker_groups[count.index], - "asg_max_size", - local.workers_group_defaults["asg_max_size"], - ) - min_size = lookup( - var.worker_groups[count.index], - "asg_min_size", - local.workers_group_defaults["asg_min_size"], - ) - force_delete = lookup( - var.worker_groups[count.index], - "asg_force_delete", - local.workers_group_defaults["asg_force_delete"], - ) - target_group_arns = lookup( - var.worker_groups[count.index], - "target_group_arns", - local.workers_group_defaults["target_group_arns"] - ) - load_balancers = lookup( - var.worker_groups[count.index], - "load_balancers", - local.workers_group_defaults["load_balancers"] - ) - service_linked_role_arn = lookup( - var.worker_groups[count.index], - "service_linked_role_arn", - local.workers_group_defaults["service_linked_role_arn"], - ) - launch_configuration = aws_launch_configuration.workers.*.id[count.index] - vpc_zone_identifier = lookup( - var.worker_groups[count.index], - "subnets", - local.workers_group_defaults["subnets"] - ) - protect_from_scale_in = lookup( - var.worker_groups[count.index], - "protect_from_scale_in", - local.workers_group_defaults["protect_from_scale_in"], - ) - suspended_processes = lookup( - var.worker_groups[count.index], - "suspended_processes", - local.workers_group_defaults["suspended_processes"] - ) - enabled_metrics = lookup( - var.worker_groups[count.index], - "enabled_metrics", - local.workers_group_defaults["enabled_metrics"] - ) - placement_group = lookup( - var.worker_groups[count.index], - "placement_group", - local.workers_group_defaults["placement_group"], - ) - termination_policies = lookup( - var.worker_groups[count.index], - "termination_policies", - local.workers_group_defaults["termination_policies"] - ) - max_instance_lifetime = lookup( - var.worker_groups[count.index], - "max_instance_lifetime", - local.workers_group_defaults["max_instance_lifetime"], - ) - default_cooldown = lookup( - var.worker_groups[count.index], - "default_cooldown", - local.workers_group_defaults["default_cooldown"] - ) - health_check_type = lookup( - var.worker_groups[count.index], - "health_check_type", - local.workers_group_defaults["health_check_type"] - ) - health_check_grace_period = lookup( - var.worker_groups[count.index], - "health_check_grace_period", - local.workers_group_defaults["health_check_grace_period"] - ) - capacity_rebalance = lookup( - var.worker_groups[count.index], - "capacity_rebalance", - local.workers_group_defaults["capacity_rebalance"] - ) - - dynamic "initial_lifecycle_hook" { - for_each = var.worker_create_initial_lifecycle_hooks ? lookup(var.worker_groups[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) : [] - content { - name = initial_lifecycle_hook.value["name"] - lifecycle_transition = initial_lifecycle_hook.value["lifecycle_transition"] - notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) - heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) - notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) - role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) - default_result = lookup(initial_lifecycle_hook.value, "default_result", null) - } - } - - dynamic "warm_pool" { - for_each = lookup(var.worker_groups[count.index], "warm_pool", null) != null ? [lookup(var.worker_groups[count.index], "warm_pool")] : [] - - content { - pool_state = lookup(warm_pool.value, "pool_state", null) - min_size = lookup(warm_pool.value, "min_size", null) - max_group_prepared_capacity = lookup(warm_pool.value, "max_group_prepared_capacity", null) - } - } - - dynamic "tag" { - for_each = concat( - [ - { - "key" = "Name" - "value" = "${local.cluster_name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg" - "propagate_at_launch" = true - }, - { - "key" = "kubernetes.io/cluster/${local.cluster_name}" - "value" = "owned" - "propagate_at_launch" = true - }, - { - "key" = "k8s.io/cluster/${local.cluster_name}" - "value" = "owned" - "propagate_at_launch" = true - }, - ], - [ - for tag_key, tag_value in var.tags : - { - "key" = tag_key, - "value" = tag_value, - "propagate_at_launch" = "true" - } - if tag_key != "Name" && !contains([for tag in lookup(var.worker_groups[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) - ], - lookup( - var.worker_groups[count.index], - "tags", - local.workers_group_defaults["tags"] - ) - ) - content { - key = tag.value.key - value = tag.value.value - propagate_at_launch = tag.value.propagate_at_launch - } - } - - # logic duplicated in workers_launch_template.tf - dynamic "instance_refresh" { - for_each = lookup(var.worker_groups[count.index], - "instance_refresh_enabled", - local.workers_group_defaults["instance_refresh_enabled"]) ? [1] : [] - content { - strategy = lookup( - var.worker_groups[count.index], "instance_refresh_strategy", - local.workers_group_defaults["instance_refresh_strategy"] - ) - preferences { - instance_warmup = lookup( - var.worker_groups[count.index], "instance_refresh_instance_warmup", - local.workers_group_defaults["instance_refresh_instance_warmup"] - ) - min_healthy_percentage = lookup( - var.worker_groups[count.index], "instance_refresh_min_healthy_percentage", - local.workers_group_defaults["instance_refresh_min_healthy_percentage"] - ) - } - triggers = lookup( - var.worker_groups[count.index], "instance_refresh_triggers", - local.workers_group_defaults["instance_refresh_triggers"] - ) - } - } - - lifecycle { - create_before_destroy = true - ignore_changes = [desired_capacity] - } -} - -resource "aws_launch_configuration" "workers" { - count = var.create_eks ? local.worker_group_launch_configuration_count : 0 - - name_prefix = "${local.cluster_name}-${lookup(var.worker_groups[count.index], "name", count.index)}" - associate_public_ip_address = lookup( - var.worker_groups[count.index], - "public_ip", - local.workers_group_defaults["public_ip"], - ) - security_groups = flatten([ - local.worker_security_group_id, - var.worker_additional_security_group_ids, - lookup( - var.worker_groups[count.index], - "additional_security_group_ids", - local.workers_group_defaults["additional_security_group_ids"] - ) - ]) - iam_instance_profile = coalescelist( - aws_iam_instance_profile.workers.*.id, - data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.name, - )[count.index] - image_id = lookup( - var.worker_groups[count.index], - "ami_id", - lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? local.default_ami_id_windows : local.default_ami_id_linux, - ) - instance_type = lookup( - var.worker_groups[count.index], - "instance_type", - local.workers_group_defaults["instance_type"], - ) - key_name = lookup( - var.worker_groups[count.index], - "key_name", - local.workers_group_defaults["key_name"], - ) - user_data_base64 = base64encode(local.launch_configuration_userdata_rendered[count.index]) - ebs_optimized = lookup( - var.worker_groups[count.index], - "ebs_optimized", - !contains( - local.ebs_optimized_not_supported, - lookup( - var.worker_groups[count.index], - "instance_type", - local.workers_group_defaults["instance_type"] - ) - ) - ) - enable_monitoring = lookup( - var.worker_groups[count.index], - "enable_monitoring", - local.workers_group_defaults["enable_monitoring"], - ) - spot_price = lookup( - var.worker_groups[count.index], - "spot_price", - local.workers_group_defaults["spot_price"], - ) - placement_tenancy = lookup( - var.worker_groups[count.index], - "placement_tenancy", - local.workers_group_defaults["placement_tenancy"], - ) - - metadata_options { - http_endpoint = lookup( - var.worker_groups[count.index], - "metadata_http_endpoint", - local.workers_group_defaults["metadata_http_endpoint"], - ) - http_tokens = lookup( - var.worker_groups[count.index], - "metadata_http_tokens", - local.workers_group_defaults["metadata_http_tokens"], - ) - http_put_response_hop_limit = lookup( - var.worker_groups[count.index], - "metadata_http_put_response_hop_limit", - local.workers_group_defaults["metadata_http_put_response_hop_limit"], - ) - } - - root_block_device { - encrypted = lookup( - var.worker_groups[count.index], - "root_encrypted", - local.workers_group_defaults["root_encrypted"], - ) - volume_size = lookup( - var.worker_groups[count.index], - "root_volume_size", - local.workers_group_defaults["root_volume_size"], - ) - volume_type = lookup( - var.worker_groups[count.index], - "root_volume_type", - local.workers_group_defaults["root_volume_type"], - ) - iops = lookup( - var.worker_groups[count.index], - "root_iops", - local.workers_group_defaults["root_iops"], - ) - throughput = lookup( - var.worker_groups[count.index], - "root_volume_throughput", - local.workers_group_defaults["root_volume_throughput"], - ) - delete_on_termination = true - } - - dynamic "ebs_block_device" { - for_each = lookup(var.worker_groups[count.index], "additional_ebs_volumes", local.workers_group_defaults["additional_ebs_volumes"]) - - content { - device_name = ebs_block_device.value.block_device_name - volume_size = lookup( - ebs_block_device.value, - "volume_size", - local.workers_group_defaults["root_volume_size"], - ) - volume_type = lookup( - ebs_block_device.value, - "volume_type", - local.workers_group_defaults["root_volume_type"], - ) - iops = lookup( - ebs_block_device.value, - "iops", - local.workers_group_defaults["root_iops"], - ) - throughput = lookup( - ebs_block_device.value, - "throughput", - local.workers_group_defaults["root_volume_throughput"], - ) - encrypted = lookup( - ebs_block_device.value, - "encrypted", - local.workers_group_defaults["root_encrypted"], - ) - snapshot_id = lookup( - ebs_block_device.value, - "snapshot_id", - local.workers_group_defaults["snapshot_id"], - ) - delete_on_termination = lookup(ebs_block_device.value, "delete_on_termination", true) - } - } - - lifecycle { - create_before_destroy = true - } - - # Prevent premature access of security group roles and policies by pods that - # require permissions on create/destroy that depend on workers. - depends_on = [ - aws_security_group_rule.workers_egress_internet, - aws_security_group_rule.workers_ingress_self, - aws_security_group_rule.workers_ingress_cluster, - aws_security_group_rule.workers_ingress_cluster_kubelet, - aws_security_group_rule.workers_ingress_cluster_https, - aws_security_group_rule.workers_ingress_cluster_primary, - aws_security_group_rule.cluster_primary_ingress_workers, - aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy, - aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy, - aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly, - aws_iam_role_policy_attachment.workers_additional_policies - ] -} - -resource "aws_security_group" "workers" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 - - name_prefix = var.cluster_name - description = "Security group for all nodes in the cluster." - vpc_id = var.vpc_id - tags = merge( - var.tags, - { - "Name" = "${var.cluster_name}-eks_worker_sg" - "kubernetes.io/cluster/${var.cluster_name}" = "owned" - }, - ) -} - -resource "aws_security_group_rule" "workers_egress_internet" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 - - description = "Allow nodes all egress to the Internet." - protocol = "-1" - security_group_id = local.worker_security_group_id - cidr_blocks = var.workers_egress_cidrs - from_port = 0 - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "workers_ingress_self" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 - - description = "Allow node to communicate with each other." - protocol = "-1" - security_group_id = local.worker_security_group_id - source_security_group_id = local.worker_security_group_id - from_port = 0 - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "workers_ingress_cluster" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 - - description = "Allow workers pods to receive communication from the cluster control plane." - protocol = "tcp" - security_group_id = local.worker_security_group_id - source_security_group_id = local.cluster_security_group_id - from_port = var.worker_sg_ingress_from_port - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { - count = var.worker_create_security_group && var.create_eks ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 - - description = "Allow workers Kubelets to receive communication from the cluster control plane." - protocol = "tcp" - security_group_id = local.worker_security_group_id - source_security_group_id = local.cluster_security_group_id - from_port = 10250 - to_port = 10250 - type = "ingress" -} - -resource "aws_security_group_rule" "workers_ingress_cluster_https" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 - - description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane." - protocol = "tcp" - security_group_id = local.worker_security_group_id - source_security_group_id = local.cluster_security_group_id - from_port = 443 - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "workers_ingress_cluster_primary" { - count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.create_eks ? 1 : 0 - - description = "Allow pods running on workers to receive communication from cluster primary security group (e.g. Fargate pods)." - protocol = "all" - security_group_id = local.worker_security_group_id - source_security_group_id = local.cluster_primary_security_group_id - from_port = 0 - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "cluster_primary_ingress_workers" { - count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.create_eks ? 1 : 0 - - description = "Allow pods running on workers to send communication to cluster primary security group (e.g. Fargate pods)." - protocol = "all" - security_group_id = local.cluster_primary_security_group_id - source_security_group_id = local.worker_security_group_id - from_port = 0 - to_port = 65535 - type = "ingress" -} - -resource "aws_iam_role" "workers" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - - name_prefix = var.workers_role_name != "" ? null : local.cluster_name - name = var.workers_role_name != "" ? var.workers_role_name : null - assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json - permissions_boundary = var.permissions_boundary - path = var.iam_path - force_detach_policies = true - - tags = var.tags -} - -resource "aws_iam_instance_profile" "workers" { - count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_launch_configuration_count : 0 - - name_prefix = local.cluster_name - role = lookup( - var.worker_groups[count.index], - "iam_role_id", - local.default_iam_role_id, - ) - path = var.iam_path - - tags = var.tags - - lifecycle { - create_before_destroy = true - } -} - -resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - - policy_arn = "${local.policy_arn_prefix}/AmazonEKSWorkerNodePolicy" - role = aws_iam_role.workers[0].name -} - -resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { - count = var.manage_worker_iam_resources && var.attach_worker_cni_policy && var.create_eks ? 1 : 0 - - policy_arn = "${local.policy_arn_prefix}/AmazonEKS_CNI_Policy" - role = aws_iam_role.workers[0].name -} - -resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - - policy_arn = "${local.policy_arn_prefix}/AmazonEC2ContainerRegistryReadOnly" - role = aws_iam_role.workers[0].name -} - -resource "aws_iam_role_policy_attachment" "workers_additional_policies" { - count = var.manage_worker_iam_resources && var.create_eks ? length(var.workers_additional_policies) : 0 - - role = aws_iam_role.workers[0].name - policy_arn = var.workers_additional_policies[count.index] -} diff --git a/workers_launch_template.tf b/workers_launch_template.tf deleted file mode 100644 index df2928dc5d..0000000000 --- a/workers_launch_template.tf +++ /dev/null @@ -1,647 +0,0 @@ -# Worker Groups using Launch Templates - -resource "aws_autoscaling_group" "workers_launch_template" { - count = var.create_eks ? local.worker_group_launch_template_count : 0 - - name_prefix = join( - "-", - compact( - [ - local.cluster_name, - lookup(var.worker_groups_launch_template[count.index], "name", count.index) - ] - ) - ) - desired_capacity = lookup( - var.worker_groups_launch_template[count.index], - "asg_desired_capacity", - local.workers_group_defaults["asg_desired_capacity"], - ) - max_size = lookup( - var.worker_groups_launch_template[count.index], - "asg_max_size", - local.workers_group_defaults["asg_max_size"], - ) - min_size = lookup( - var.worker_groups_launch_template[count.index], - "asg_min_size", - local.workers_group_defaults["asg_min_size"], - ) - force_delete = lookup( - var.worker_groups_launch_template[count.index], - "asg_force_delete", - local.workers_group_defaults["asg_force_delete"], - ) - target_group_arns = lookup( - var.worker_groups_launch_template[count.index], - "target_group_arns", - local.workers_group_defaults["target_group_arns"] - ) - load_balancers = lookup( - var.worker_groups_launch_template[count.index], - "load_balancers", - local.workers_group_defaults["load_balancers"] - ) - service_linked_role_arn = lookup( - var.worker_groups_launch_template[count.index], - "service_linked_role_arn", - local.workers_group_defaults["service_linked_role_arn"], - ) - vpc_zone_identifier = lookup( - var.worker_groups_launch_template[count.index], - "subnets", - local.workers_group_defaults["subnets"] - ) - protect_from_scale_in = lookup( - var.worker_groups_launch_template[count.index], - "protect_from_scale_in", - local.workers_group_defaults["protect_from_scale_in"], - ) - suspended_processes = lookup( - var.worker_groups_launch_template[count.index], - "suspended_processes", - local.workers_group_defaults["suspended_processes"] - ) - enabled_metrics = lookup( - var.worker_groups_launch_template[count.index], - "enabled_metrics", - local.workers_group_defaults["enabled_metrics"] - ) - placement_group = lookup( - var.worker_groups_launch_template[count.index], - "placement_group", - local.workers_group_defaults["placement_group"], - ) - termination_policies = lookup( - var.worker_groups_launch_template[count.index], - "termination_policies", - local.workers_group_defaults["termination_policies"] - ) - max_instance_lifetime = lookup( - var.worker_groups_launch_template[count.index], - "max_instance_lifetime", - local.workers_group_defaults["max_instance_lifetime"], - ) - default_cooldown = lookup( - var.worker_groups_launch_template[count.index], - "default_cooldown", - local.workers_group_defaults["default_cooldown"] - ) - health_check_type = lookup( - var.worker_groups_launch_template[count.index], - "health_check_type", - local.workers_group_defaults["health_check_type"] - ) - health_check_grace_period = lookup( - var.worker_groups_launch_template[count.index], - "health_check_grace_period", - local.workers_group_defaults["health_check_grace_period"] - ) - capacity_rebalance = lookup( - var.worker_groups_launch_template[count.index], - "capacity_rebalance", - local.workers_group_defaults["capacity_rebalance"] - ) - - dynamic "mixed_instances_policy" { - iterator = item - for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? [var.worker_groups_launch_template[count.index]] : [] - - content { - instances_distribution { - on_demand_allocation_strategy = lookup( - item.value, - "on_demand_allocation_strategy", - "prioritized", - ) - on_demand_base_capacity = lookup( - item.value, - "on_demand_base_capacity", - local.workers_group_defaults["on_demand_base_capacity"], - ) - on_demand_percentage_above_base_capacity = lookup( - item.value, - "on_demand_percentage_above_base_capacity", - local.workers_group_defaults["on_demand_percentage_above_base_capacity"], - ) - spot_allocation_strategy = lookup( - item.value, - "spot_allocation_strategy", - local.workers_group_defaults["spot_allocation_strategy"], - ) - spot_instance_pools = lookup( - item.value, - "spot_instance_pools", - local.workers_group_defaults["spot_instance_pools"], - ) - spot_max_price = lookup( - item.value, - "spot_max_price", - local.workers_group_defaults["spot_max_price"], - ) - } - - launch_template { - launch_template_specification { - launch_template_id = aws_launch_template.workers_launch_template.*.id[count.index] - version = lookup( - var.worker_groups_launch_template[count.index], - "launch_template_version", - lookup( - var.worker_groups_launch_template[count.index], - "launch_template_version", - local.workers_group_defaults["launch_template_version"] - ) == "$Latest" - ? aws_launch_template.workers_launch_template.*.latest_version[count.index] - : aws_launch_template.workers_launch_template.*.default_version[count.index] - ) - } - - dynamic "override" { - for_each = lookup( - var.worker_groups_launch_template[count.index], - "override_instance_types", - local.workers_group_defaults["override_instance_types"] - ) - - content { - instance_type = override.value - } - } - } - } - } - - dynamic "launch_template" { - iterator = item - for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? [] : [var.worker_groups_launch_template[count.index]] - - content { - id = aws_launch_template.workers_launch_template.*.id[count.index] - version = lookup( - var.worker_groups_launch_template[count.index], - "launch_template_version", - lookup( - var.worker_groups_launch_template[count.index], - "launch_template_version", - local.workers_group_defaults["launch_template_version"] - ) == "$Latest" - ? aws_launch_template.workers_launch_template.*.latest_version[count.index] - : aws_launch_template.workers_launch_template.*.default_version[count.index] - ) - } - } - - dynamic "initial_lifecycle_hook" { - for_each = var.worker_create_initial_lifecycle_hooks ? lookup(var.worker_groups_launch_template[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) : [] - content { - name = initial_lifecycle_hook.value["name"] - lifecycle_transition = initial_lifecycle_hook.value["lifecycle_transition"] - notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) - heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) - notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) - role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) - default_result = lookup(initial_lifecycle_hook.value, "default_result", null) - } - } - - dynamic "warm_pool" { - for_each = lookup(var.worker_groups_launch_template[count.index], "warm_pool", null) != null ? [lookup(var.worker_groups_launch_template[count.index], "warm_pool")] : [] - - content { - pool_state = lookup(warm_pool.value, "pool_state", null) - min_size = lookup(warm_pool.value, "min_size", null) - max_group_prepared_capacity = lookup(warm_pool.value, "max_group_prepared_capacity", null) - } - } - - dynamic "tag" { - for_each = concat( - [ - { - "key" = "Name" - "value" = "${local.cluster_name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}-eks_asg" - "propagate_at_launch" = true - }, - { - "key" = "kubernetes.io/cluster/${local.cluster_name}" - "value" = "owned" - "propagate_at_launch" = true - }, - ], - [ - for tag_key, tag_value in var.tags : - tomap({ - key = tag_key - value = tag_value - propagate_at_launch = "true" - }) - if tag_key != "Name" && !contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) - ], - lookup( - var.worker_groups_launch_template[count.index], - "tags", - local.workers_group_defaults["tags"] - ) - ) - content { - key = tag.value.key - value = tag.value.value - propagate_at_launch = tag.value.propagate_at_launch - } - } - - # logic duplicated in workers.tf - dynamic "instance_refresh" { - for_each = lookup(var.worker_groups_launch_template[count.index], - "instance_refresh_enabled", - local.workers_group_defaults["instance_refresh_enabled"]) ? [1] : [] - content { - strategy = lookup( - var.worker_groups_launch_template[count.index], "instance_refresh_strategy", - local.workers_group_defaults["instance_refresh_strategy"] - ) - preferences { - instance_warmup = lookup( - var.worker_groups_launch_template[count.index], "instance_refresh_instance_warmup", - local.workers_group_defaults["instance_refresh_instance_warmup"] - ) - min_healthy_percentage = lookup( - var.worker_groups_launch_template[count.index], "instance_refresh_min_healthy_percentage", - local.workers_group_defaults["instance_refresh_min_healthy_percentage"] - ) - } - triggers = lookup( - var.worker_groups_launch_template[count.index], "instance_refresh_triggers", - local.workers_group_defaults["instance_refresh_triggers"] - ) - } - } - - lifecycle { - create_before_destroy = true - ignore_changes = [desired_capacity] - } -} - -resource "aws_launch_template" "workers_launch_template" { - count = var.create_eks ? (local.worker_group_launch_template_count) : 0 - - name_prefix = "${local.cluster_name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}" - - update_default_version = lookup( - var.worker_groups_launch_template[count.index], - "update_default_version", - local.workers_group_defaults["update_default_version"], - ) - - network_interfaces { - associate_public_ip_address = lookup( - var.worker_groups_launch_template[count.index], - "public_ip", - local.workers_group_defaults["public_ip"], - ) - delete_on_termination = lookup( - var.worker_groups_launch_template[count.index], - "eni_delete", - local.workers_group_defaults["eni_delete"], - ) - interface_type = lookup( - var.worker_groups_launch_template[count.index], - "interface_type", - local.workers_group_defaults["interface_type"], - ) - security_groups = flatten([ - local.worker_security_group_id, - var.worker_additional_security_group_ids, - lookup( - var.worker_groups_launch_template[count.index], - "additional_security_group_ids", - local.workers_group_defaults["additional_security_group_ids"], - ), - ]) - } - - iam_instance_profile { - name = coalescelist( - aws_iam_instance_profile.workers_launch_template.*.name, - data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.name, - )[count.index] - } - - enclave_options { - enabled = lookup( - var.worker_groups_launch_template[count.index], - "enclave_support", - local.workers_group_defaults["enclave_support"], - ) - } - - image_id = lookup( - var.worker_groups_launch_template[count.index], - "ami_id", - lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? local.default_ami_id_windows : local.default_ami_id_linux, - ) - instance_type = lookup( - var.worker_groups_launch_template[count.index], - "instance_type", - local.workers_group_defaults["instance_type"], - ) - - dynamic "elastic_inference_accelerator" { - for_each = lookup( - var.worker_groups_launch_template[count.index], - "elastic_inference_accelerator", - local.workers_group_defaults["elastic_inference_accelerator"] - ) != null ? [lookup(var.worker_groups_launch_template[count.index], "elastic_inference_accelerator", local.workers_group_defaults["elastic_inference_accelerator"])] : [] - content { - type = elastic_inference_accelerator.value - } - } - - key_name = lookup( - var.worker_groups_launch_template[count.index], - "key_name", - local.workers_group_defaults["key_name"], - ) - user_data = base64encode( - local.launch_template_userdata_rendered[count.index], - ) - - ebs_optimized = lookup( - var.worker_groups_launch_template[count.index], - "ebs_optimized", - !contains( - local.ebs_optimized_not_supported, - lookup( - var.worker_groups_launch_template[count.index], - "instance_type", - local.workers_group_defaults["instance_type"], - ) - ) - ) - - metadata_options { - http_endpoint = lookup( - var.worker_groups_launch_template[count.index], - "metadata_http_endpoint", - local.workers_group_defaults["metadata_http_endpoint"], - ) - http_tokens = lookup( - var.worker_groups_launch_template[count.index], - "metadata_http_tokens", - local.workers_group_defaults["metadata_http_tokens"], - ) - http_put_response_hop_limit = lookup( - var.worker_groups_launch_template[count.index], - "metadata_http_put_response_hop_limit", - local.workers_group_defaults["metadata_http_put_response_hop_limit"], - ) - } - - dynamic "credit_specification" { - for_each = lookup( - var.worker_groups_launch_template[count.index], - "cpu_credits", - local.workers_group_defaults["cpu_credits"] - ) != null ? [lookup(var.worker_groups_launch_template[count.index], "cpu_credits", local.workers_group_defaults["cpu_credits"])] : [] - content { - cpu_credits = credit_specification.value - } - } - - monitoring { - enabled = lookup( - var.worker_groups_launch_template[count.index], - "enable_monitoring", - local.workers_group_defaults["enable_monitoring"], - ) - } - - dynamic "placement" { - for_each = lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"]) != null ? [lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])] : [] - - content { - tenancy = lookup( - var.worker_groups_launch_template[count.index], - "launch_template_placement_tenancy", - local.workers_group_defaults["launch_template_placement_tenancy"], - ) - group_name = placement.value - } - } - - dynamic "instance_market_options" { - for_each = lookup(var.worker_groups_launch_template[count.index], "market_type", null) == null ? [] : tolist([lookup(var.worker_groups_launch_template[count.index], "market_type", null)]) - content { - market_type = instance_market_options.value - } - } - - block_device_mappings { - device_name = lookup( - var.worker_groups_launch_template[count.index], - "root_block_device_name", - lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? local.workers_group_defaults["root_block_device_name_windows"] : local.workers_group_defaults["root_block_device_name"], - ) - - ebs { - volume_size = lookup( - var.worker_groups_launch_template[count.index], - "root_volume_size", - local.workers_group_defaults["root_volume_size"], - ) - volume_type = lookup( - var.worker_groups_launch_template[count.index], - "root_volume_type", - local.workers_group_defaults["root_volume_type"], - ) - iops = lookup( - var.worker_groups_launch_template[count.index], - "root_iops", - local.workers_group_defaults["root_iops"], - ) - throughput = lookup( - var.worker_groups_launch_template[count.index], - "root_volume_throughput", - local.workers_group_defaults["root_volume_throughput"], - ) - encrypted = lookup( - var.worker_groups_launch_template[count.index], - "root_encrypted", - local.workers_group_defaults["root_encrypted"], - ) - kms_key_id = lookup( - var.worker_groups_launch_template[count.index], - "root_kms_key_id", - local.workers_group_defaults["root_kms_key_id"], - ) - delete_on_termination = true - } - } - - dynamic "block_device_mappings" { - for_each = lookup(var.worker_groups_launch_template[count.index], "additional_ebs_volumes", local.workers_group_defaults["additional_ebs_volumes"]) - content { - device_name = block_device_mappings.value.block_device_name - - ebs { - volume_size = lookup( - block_device_mappings.value, - "volume_size", - local.workers_group_defaults["root_volume_size"], - ) - volume_type = lookup( - block_device_mappings.value, - "volume_type", - local.workers_group_defaults["root_volume_type"], - ) - iops = lookup( - block_device_mappings.value, - "iops", - local.workers_group_defaults["root_iops"], - ) - throughput = lookup( - block_device_mappings.value, - "throughput", - local.workers_group_defaults["root_volume_throughput"], - ) - encrypted = lookup( - block_device_mappings.value, - "encrypted", - local.workers_group_defaults["root_encrypted"], - ) - kms_key_id = lookup( - block_device_mappings.value, - "kms_key_id", - local.workers_group_defaults["root_kms_key_id"], - ) - snapshot_id = lookup( - block_device_mappings.value, - "snapshot_id", - local.workers_group_defaults["snapshot_id"], - ) - delete_on_termination = lookup(block_device_mappings.value, "delete_on_termination", true) - } - } - - } - - dynamic "block_device_mappings" { - for_each = lookup(var.worker_groups_launch_template[count.index], "additional_instance_store_volumes", local.workers_group_defaults["additional_instance_store_volumes"]) - content { - device_name = block_device_mappings.value.block_device_name - virtual_name = lookup( - block_device_mappings.value, - "virtual_name", - local.workers_group_defaults["instance_store_virtual_name"], - ) - } - } - - tag_specifications { - resource_type = "volume" - - tags = merge( - { - "Name" = "${local.cluster_name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}-eks_asg" - }, - var.tags, - { - for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : - tag["key"] => tag["value"] - if tag["key"] != "Name" && tag["propagate_at_launch"] - } - ) - } - - tag_specifications { - resource_type = "instance" - - tags = merge( - { - "Name" = "${local.cluster_name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}-eks_asg" - }, - { for tag_key, tag_value in var.tags : - tag_key => tag_value - if tag_key != "Name" && !contains([for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : tag["key"]], tag_key) - } - ) - } - - tag_specifications { - resource_type = "network-interface" - - tags = merge( - { - "Name" = "${local.cluster_name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}-eks_asg" - }, - var.tags, - { - for tag in lookup(var.worker_groups_launch_template[count.index], "tags", local.workers_group_defaults["tags"]) : - tag["key"] => tag["value"] - if tag["key"] != "Name" && tag["propagate_at_launch"] - } - ) - } - - tags = var.tags - - lifecycle { - create_before_destroy = true - } - - # Prevent premature access of security group roles and policies by pods that - # require permissions on create/destroy that depend on workers. - depends_on = [ - aws_security_group_rule.workers_egress_internet, - aws_security_group_rule.workers_ingress_self, - aws_security_group_rule.workers_ingress_cluster, - aws_security_group_rule.workers_ingress_cluster_kubelet, - aws_security_group_rule.workers_ingress_cluster_https, - aws_security_group_rule.workers_ingress_cluster_primary, - aws_security_group_rule.cluster_primary_ingress_workers, - aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy, - aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy, - aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly, - aws_iam_role_policy_attachment.workers_additional_policies - ] -} - -resource "aws_iam_instance_profile" "workers_launch_template" { - count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_launch_template_count : 0 - - name_prefix = local.cluster_name - role = lookup( - var.worker_groups_launch_template[count.index], - "iam_role_id", - local.default_iam_role_id, - ) - path = var.iam_path - - tags = var.tags - - lifecycle { - create_before_destroy = true - } -} From be86c0b898c34943e898e2ecd4994bb7904663ff Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 5 Jan 2022 14:56:42 -0500 Subject: [PATCH 426/962] fix: Update preset rule on semantic-release to use conventional commits (#1736) --- .github/workflows/release.yml | 2 ++ .pre-commit-config.yaml | 4 ++-- .releaserc.json | 17 +++++++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 63efd61394..e5529a950d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,7 @@ on: - '**/*.tpl' - '**/*.py' - '**/*.tf' + - '.github/workflows/release.yml' jobs: release: @@ -29,5 +30,6 @@ jobs: extra_plugins: | @semantic-release/changelog@6.0.0 @semantic-release/git@10.0.0 + conventional-changelog-conventionalcommits@4.6.3 env: GITHUB_TOKEN: ${{ secrets.SEMANTIC_RELEASE_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 006410918f..093121e086 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.62.0 + rev: v1.62.3 hooks: - id: terraform_fmt - id: terraform_validate @@ -23,7 +23,7 @@ repos: - '--args=--only=terraform_standard_module_structure' - '--args=--only=terraform_workspace_remote' - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.1.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer diff --git a/.releaserc.json b/.releaserc.json index 6e39031cf2..66b3eefd65 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -5,13 +5,22 @@ ], "ci": false, "plugins": [ - "@semantic-release/commit-analyzer", - "@semantic-release/release-notes-generator", + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits" + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits" + } + ], [ "@semantic-release/github", { - "successComment": - "This ${issue.pull_request ? 'PR is included' : 'issue has been resolved'} in version ${nextRelease.version} :tada:", + "successComment": "This ${issue.pull_request ? 'PR is included' : 'issue has been resolved'} in version ${nextRelease.version} :tada:", "labels": false, "releasedLabels": false } From d71ef01c37f777d28cbf6c6e8dbc161de073d6b1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 5 Jan 2022 19:57:24 +0000 Subject: [PATCH 427/962] chore(release): version 18.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [18.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.24.0...v18.0.0) (2022-01-05) ### ⚠ BREAKING CHANGES * Removed support for launch configuration and replace `count` with `for_each` (#1680) ### Features * Removed support for launch configuration and replace `count` with `for_each` ([#1680](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1680)) ([ee9f0c6](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ee9f0c646a45ca9baa6174a036d1e09bcccb87b1)) ### Bug Fixes * Update preset rule on semantic-release to use conventional commits ([#1736](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1736)) ([be86c0b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/be86c0b898c34943e898e2ecd4994bb7904663ff)) --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15937e80c2..0f07294de2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ All notable changes to this project will be documented in this file. +## [18.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.24.0...v18.0.0) (2022-01-05) + + +### ⚠ BREAKING CHANGES + +* Removed support for launch configuration and replace `count` with `for_each` (#1680) + +### Features + +* Removed support for launch configuration and replace `count` with `for_each` ([#1680](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1680)) ([ee9f0c6](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ee9f0c646a45ca9baa6174a036d1e09bcccb87b1)) + + +### Bug Fixes + +* Update preset rule on semantic-release to use conventional commits ([#1736](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1736)) ([be86c0b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/be86c0b898c34943e898e2ecd4994bb7904663ff)) + # [17.24.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.23.0...v17.24.0) (2021-11-22) From a2c7caac9f01ef167994d8b62afb5f997d0fac66 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 6 Jan 2022 06:27:04 -0500 Subject: [PATCH 428/962] fix: Correct conditional map for cluster security group additional rules (#1738) --- README.md | 2 +- examples/eks_managed_node_group/main.tf | 11 +++++++++++ main.tf | 2 +- variables.tf | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c93a37a8d8..b519f58933 100644 --- a/README.md +++ b/README.md @@ -695,7 +695,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | -| [cluster\_security\_group\_additional\_rules](#input\_cluster\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created | `map(any)` | `{}` | no | +| [cluster\_security\_group\_additional\_rules](#input\_cluster\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created | `any` | `{}` | no | | [cluster\_security\_group\_description](#input\_cluster\_security\_group\_description) | Description of the cluster security group created | `string` | `"EKS cluster security group"` | no | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Existing security group ID to be attached to the cluster. Required if `create_cluster_security_group` = `false` | `string` | `""` | no | | [cluster\_security\_group\_name](#input\_cluster\_security\_group\_name) | Name to use on cluster security group created | `string` | `null` | no | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index fb801c1eb1..68d3868a56 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -44,6 +44,17 @@ module "eks" { resources = ["secrets"] }] + cluster_security_group_additional_rules = { + admin_access = { + description = "Admin ingress to Kubernetes API" + cidr_blocks = ["10.97.0.0/30"] + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "ingress" + } + } + vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets diff --git a/main.tf b/main.tf index 286e23b5a4..4817f96047 100644 --- a/main.tf +++ b/main.tf @@ -119,7 +119,7 @@ resource "aws_security_group" "cluster" { } resource "aws_security_group_rule" "cluster" { - for_each = local.create_cluster_sg ? merge(local.cluster_security_group_rules, var.cluster_security_group_additional_rules) : {} + for_each = { for k, v in merge(local.cluster_security_group_rules, var.cluster_security_group_additional_rules) : k => v if local.create_cluster_sg } # Required security_group_id = aws_security_group.cluster[0].id diff --git a/variables.tf b/variables.tf index 12ff69ee33..6c69760d52 100644 --- a/variables.tf +++ b/variables.tf @@ -153,7 +153,7 @@ variable "cluster_security_group_description" { variable "cluster_security_group_additional_rules" { description = "List of additional security group rules to add to the cluster security group created" - type = map(any) + type = any default = {} } From 3d151b2c85aa964ca524e16a83192f818f16e514 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 6 Jan 2022 11:27:47 +0000 Subject: [PATCH 429/962] chore(release): version 18.0.1 [skip ci] ### [18.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.0...v18.0.1) (2022-01-06) ### Bug Fixes * Correct conditional map for cluster security group additional rules ([#1738](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1738)) ([a2c7caa](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a2c7caac9f01ef167994d8b62afb5f997d0fac66)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f07294de2..5f9c3086ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.0...v18.0.1) (2022-01-06) + + +### Bug Fixes + +* Correct conditional map for cluster security group additional rules ([#1738](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1738)) ([a2c7caa](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a2c7caac9f01ef167994d8b62afb5f997d0fac66)) + ## [18.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.24.0...v18.0.0) (2022-01-05) From 89218279d4439110439ca4cb8ac94575ab92b042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Magalh=C3=A3es?= <4622652+pjrm@users.noreply.github.com> Date: Thu, 6 Jan 2022 20:10:10 +0000 Subject: [PATCH 430/962] fix: Change variable "node_security_group_additional_rules" from type map(any) to any (#1747) --- README.md | 2 +- variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b519f58933..4b63771cc4 100644 --- a/README.md +++ b/README.md @@ -723,7 +723,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | -| [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created | `map(any)` | `{}` | no | +| [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created | `any` | `{}` | no | | [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | | [node\_security\_group\_id](#input\_node\_security\_group\_id) | ID of an existing security group to attach to the node groups created | `string` | `""` | no | | [node\_security\_group\_name](#input\_node\_security\_group\_name) | Name to use on node security group created | `string` | `null` | no | diff --git a/variables.tf b/variables.tf index 6c69760d52..1eb33bfc2c 100644 --- a/variables.tf +++ b/variables.tf @@ -199,7 +199,7 @@ variable "node_security_group_description" { variable "node_security_group_additional_rules" { description = "List of additional security group rules to add to the node security group created" - type = map(any) + type = any default = {} } From b2f6b19b71d5d68b014f79d1de54b038e089f4dd Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 6 Jan 2022 20:10:46 +0000 Subject: [PATCH 431/962] chore(release): version 18.0.2 [skip ci] ### [18.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.1...v18.0.2) (2022-01-06) ### Bug Fixes * Change variable "node_security_group_additional_rules" from type map(any) to any ([#1747](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1747)) ([8921827](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/89218279d4439110439ca4cb8ac94575ab92b042)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f9c3086ae..944bf84da1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.1...v18.0.2) (2022-01-06) + + +### Bug Fixes + +* Change variable "node_security_group_additional_rules" from type map(any) to any ([#1747](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1747)) ([8921827](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/89218279d4439110439ca4cb8ac94575ab92b042)) + ### [18.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.0...v18.0.1) (2022-01-06) From 7089c71e64dbae281435629e19d647ae6952f9ac Mon Sep 17 00:00:00 2001 From: Devin Young Date: Thu, 6 Jan 2022 15:29:15 -0500 Subject: [PATCH 432/962] fix: Remove trailing hyphen from cluster security group and iam role name prefix (#1745) --- README.md | 1 + UPGRADE-18.0.md | 2 ++ main.tf | 4 ++-- variables.tf | 6 ++++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4b63771cc4..87aa8a21c3 100644 --- a/README.md +++ b/README.md @@ -730,6 +730,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [node\_security\_group\_tags](#input\_node\_security\_group\_tags) | A map of additional tags to add to the node security group created | `map(string)` | `{}` | no | | [node\_security\_group\_use\_name\_prefix](#input\_node\_security\_group\_use\_name\_prefix) | Determines whether node security group name (`node_security_group_name`) is used as a prefix | `string` | `true` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider | `list(string)` | `[]` | no | +| [prefix\_separator](#input\_prefix\_separator) | The separator to use between the prefix and the generated timestamp for resource names | `string` | `"-"` | no | | [self\_managed\_node\_group\_defaults](#input\_self\_managed\_node\_group\_defaults) | Map of self-managed node group default configurations | `any` | `{}` | no | | [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create | `any` | `{}` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the EKS cluster (ENIs) will be provisioned along with the nodes/node groups. Node groups can be deployed within a different set of subnet IDs from within the node group configuration | `list(string)` | `[]` | no | diff --git a/UPGRADE-18.0.md b/UPGRADE-18.0.md index 0c6d56dde6..f719fcea45 100644 --- a/UPGRADE-18.0.md +++ b/UPGRADE-18.0.md @@ -28,6 +28,7 @@ Please consult the `examples` directory for reference example configurations. If - The underlying autoscaling group and launch template have been updated to more closely match that of the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module and the features it offers - The previous iteration used a count over a list of node group definitions which was prone to disruptive updates; this is now replaced with a map/for_each to align with that of the EKS managed node group and Fargate profile behaviors/style - The user data configuration supported across the module has been completely revamped. A new `_user_data` internal sub-module has been created to consolidate all user data configuration in one location which provides better support for testability (via the [`examples/user_data`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) example). The new sub-module supports nearly all possible combinations including the ability to allow users to provide their own user data template which will be rendered by the module. See the `examples/user_data` example project for the full plethora of example configuration possibilities and more details on the logic of the design can be found in the [`modules/_user_data`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/_user_data_) directory. +- Resource name changes may cause issues with existing resources. For example, security groups and IAM roles cannot be renamed, they must be recreated. Recreation of these resources may also trigger a recreation of the cluster. To use the legacy (< 18.x) resource naming convention, set `prefix_separator` to "". ## Additional changes @@ -166,6 +167,7 @@ Please consult the `examples` directory for reference example configurations. If - `cluster_addons` - `cluster_identity_providers` - `fargate_profile_defaults` + - `prefix_separator` added to support legacy behavior of not having a prefix separator - EKS Managed Node Group sub-module (was `node_groups`) - `platform` - `enable_bootstrap_user_data` diff --git a/main.tf b/main.tf index 4817f96047..3a01c64fef 100644 --- a/main.tf +++ b/main.tf @@ -107,7 +107,7 @@ resource "aws_security_group" "cluster" { count = local.create_cluster_sg ? 1 : 0 name = var.cluster_security_group_use_name_prefix ? null : local.cluster_sg_name - name_prefix = var.cluster_security_group_use_name_prefix ? "${local.cluster_sg_name}-" : null + name_prefix = var.cluster_security_group_use_name_prefix ? "${local.cluster_sg_name}${var.prefix_separator}" : null description = var.cluster_security_group_description vpc_id = var.vpc_id @@ -191,7 +191,7 @@ resource "aws_iam_role" "this" { count = var.create && var.create_iam_role ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name - name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}${var.prefix_separator}" : null path = var.iam_role_path description = var.iam_role_description diff --git a/variables.tf b/variables.tf index 1eb33bfc2c..5e715b1b8c 100644 --- a/variables.tf +++ b/variables.tf @@ -10,6 +10,12 @@ variable "tags" { default = {} } +variable "prefix_separator" { + description = "The separator to use between the prefix and the generated timestamp for resource names" + type = string + default = "-" +} + ################################################################################ # Cluster ################################################################################ From 3c6686379de9489d4c044cd13a63e2329e2ae4e1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 6 Jan 2022 20:29:46 +0000 Subject: [PATCH 433/962] chore(release): version 18.0.3 [skip ci] ### [18.0.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.2...v18.0.3) (2022-01-06) ### Bug Fixes * Remove trailing hyphen from cluster security group and iam role name prefix ([#1745](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1745)) ([7089c71](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7089c71e64dbae281435629e19d647ae6952f9ac)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 944bf84da1..a27637bba1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.0.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.2...v18.0.3) (2022-01-06) + + +### Bug Fixes + +* Remove trailing hyphen from cluster security group and iam role name prefix ([#1745](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1745)) ([7089c71](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7089c71e64dbae281435629e19d647ae6952f9ac)) + ### [18.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.1...v18.0.2) (2022-01-06) From 86b3c339a772e76239f97a9bb1f710199d1bd04a Mon Sep 17 00:00:00 2001 From: Jian Zeng Date: Fri, 7 Jan 2022 20:52:47 +0800 Subject: [PATCH 434/962] fix: Not to iterate over remote_access object in dynamic block (#1743) --- examples/eks_managed_node_group/main.tf | 4 ++++ modules/eks-managed-node-group/main.tf | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 68d3868a56..b45505a198 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -173,6 +173,10 @@ module "eks" { } ] + remote_access = { + ec2_ssh_key = "my-ssh-key" + } + update_config = { max_unavailable_percentage = 50 # or set `max_unavailable` } diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index e70e93a239..3188a8d910 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -295,10 +295,10 @@ resource "aws_eks_node_group" "this" { } dynamic "remote_access" { - for_each = var.remote_access + for_each = length(var.remote_access) > 0 ? [var.remote_access] : [] content { - ec2_ssh_key = lookup(remote_access.value, "ec2_ssh_key", null) - source_security_group_ids = lookup(remote_access.value, "source_security_group_ids", []) + ec2_ssh_key = try(remote_access.value.ec2_ssh_key, null) + source_security_group_ids = try(remote_access.value.source_security_group_ids, []) } } From b2e901c4e1ca5438959b98e9480ebe3801cd1917 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 7 Jan 2022 12:53:25 +0000 Subject: [PATCH 435/962] chore(release): version 18.0.4 [skip ci] ### [18.0.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.3...v18.0.4) (2022-01-07) ### Bug Fixes * Not to iterate over remote_access object in dynamic block ([#1743](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1743)) ([86b3c33](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/86b3c339a772e76239f97a9bb1f710199d1bd04a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a27637bba1..7a61449fb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.0.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.3...v18.0.4) (2022-01-07) + + +### Bug Fixes + +* Not to iterate over remote_access object in dynamic block ([#1743](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1743)) ([86b3c33](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/86b3c339a772e76239f97a9bb1f710199d1bd04a)) + ### [18.0.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.2...v18.0.3) (2022-01-06) From 62879dd81a69ba010f19ba9ece8392e1730b53e0 Mon Sep 17 00:00:00 2001 From: Martijn van der Ploeg <73637849+martijnvdp@users.noreply.github.com> Date: Sat, 8 Jan 2022 21:37:19 +0100 Subject: [PATCH 436/962] fix: Use the prefix_seperator var for node sg prefix (#1751) --- node_groups.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_groups.tf b/node_groups.tf index 771fa60bf8..18bd23a92f 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -106,7 +106,7 @@ resource "aws_security_group" "node" { count = local.create_node_sg ? 1 : 0 name = var.node_security_group_use_name_prefix ? null : local.node_sg_name - name_prefix = var.node_security_group_use_name_prefix ? "${local.node_sg_name}-" : null + name_prefix = var.node_security_group_use_name_prefix ? "${local.node_sg_name}${var.prefix_separator}" : null description = var.node_security_group_description vpc_id = var.vpc_id From a5bf9429bd4a58da1b64250a2c3c300cee21e21a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 8 Jan 2022 20:38:07 +0000 Subject: [PATCH 437/962] chore(release): version 18.0.5 [skip ci] ### [18.0.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.4...v18.0.5) (2022-01-08) ### Bug Fixes * Use the prefix_seperator var for node sg prefix ([#1751](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1751)) ([62879dd](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/62879dd81a69ba010f19ba9ece8392e1730b53e0)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a61449fb9..f81e8a2700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.0.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.4...v18.0.5) (2022-01-08) + + +### Bug Fixes + +* Use the prefix_seperator var for node sg prefix ([#1751](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1751)) ([62879dd](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/62879dd81a69ba010f19ba9ece8392e1730b53e0)) + ### [18.0.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.3...v18.0.4) (2022-01-07) From a1d28a721a0d877ac92bf4e7278a7b24adc26624 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 10 Jan 2022 04:22:56 -0500 Subject: [PATCH 438/962] chore: Update release configuration files to correctly use conventional-commits (#1755) --- .github/workflows/pr-title.yml | 2 +- .github/workflows/release.yml | 2 ++ .github/workflows/stale-actions.yaml | 50 ++++++++++++++-------------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index b89795d729..168011c485 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -1,4 +1,4 @@ -name: "Validate PR title" +name: 'Validate PR title' on: pull_request_target: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e5529a950d..e8a26ca6c0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,6 +16,8 @@ jobs: release: name: Release runs-on: ubuntu-latest + # Skip running release workflow on forks + if: github.repository_owner == 'terraform-aws-modules' steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/stale-actions.yaml b/.github/workflows/stale-actions.yaml index d2a5f6b96e..c09ae1d58f 100644 --- a/.github/workflows/stale-actions.yaml +++ b/.github/workflows/stale-actions.yaml @@ -1,32 +1,32 @@ -name: "Mark or close stale issues and PRs" +name: 'Mark or close stale issues and PRs' on: schedule: - - cron: "0 0 * * *" + - cron: '0 0 * * *' jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - # Staling issues and PR's - days-before-stale: 30 - stale-issue-label: stale - stale-pr-label: stale - stale-issue-message: | - This issue has been automatically marked as stale because it has been open 30 days - with no activity. Remove stale label or comment or this issue will be closed in 10 days - stale-pr-message: | - This PR has been automatically marked as stale because it has been open 30 days - with no activity. Remove stale label or comment or this PR will be closed in 10 days - # Not stale if have this labels or part of milestone - exempt-issue-labels: bug,wip,on-hold - exempt-pr-labels: bug,wip,on-hold - exempt-all-milestones: true - # Close issue operations - # Label will be automatically removed if the issues are no longer closed nor locked. - days-before-close: 10 - delete-branch: true - close-issue-message: This issue was automatically closed because of stale in 10 days - close-pr-message: This PR was automatically closed because of stale in 10 days + - uses: actions/stale@v4 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + # Staling issues and PR's + days-before-stale: 30 + stale-issue-label: stale + stale-pr-label: stale + stale-issue-message: | + This issue has been automatically marked as stale because it has been open 30 days + with no activity. Remove stale label or comment or this issue will be closed in 10 days + stale-pr-message: | + This PR has been automatically marked as stale because it has been open 30 days + with no activity. Remove stale label or comment or this PR will be closed in 10 days + # Not stale if have this labels or part of milestone + exempt-issue-labels: bug,wip,on-hold + exempt-pr-labels: bug,wip,on-hold + exempt-all-milestones: true + # Close issue operations + # Label will be automatically removed if the issues are no longer closed nor locked. + days-before-close: 10 + delete-branch: true + close-issue-message: This issue was automatically closed because of stale in 10 days + close-pr-message: This PR was automatically closed because of stale in 10 days From 7babe877758ae0287975627a0135aa8b40c29e93 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 10 Jan 2022 11:57:23 -0500 Subject: [PATCH 439/962] chore: Update documentation related to default EKS node group settings and v18.x security group changes (#1760) --- README.md | 10 ++++-- UPGRADE-18.0.md | 1 + examples/eks_managed_node_group/README.md | 4 +++ examples/eks_managed_node_group/main.tf | 40 +++++++++++++++++---- examples/eks_managed_node_group/versions.tf | 4 +++ 5 files changed, 50 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 87aa8a21c3..21c134522e 100644 --- a/README.md +++ b/README.md @@ -175,11 +175,14 @@ module "eks" { ℹ️ Only the pertinent attributes are shown for brevity -1. AWS EKS Managed Node Group can provide its own launch template and utilize the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version: +1. AWS EKS Managed Node Group can provide its own launch template and utilize the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version. By default, the module creates a launch template to ensure tags are propagated to instances, etc., so we need to disable it to use the default template provided by the AWS EKS managed node group service: ```hcl eks_managed_node_groups = { - default = {} + default = { + create_launch_template = false + launch_template_name = "" + } } ``` @@ -188,6 +191,9 @@ module "eks" { ```hcl eks_managed_node_groups = { bottlerocket_default = { + create_launch_template = false + launch_template_name = "" + ami_type = "BOTTLEROCKET_x86_64" platform = "bottlerocket" } diff --git a/UPGRADE-18.0.md b/UPGRADE-18.0.md index f719fcea45..8c1bd66983 100644 --- a/UPGRADE-18.0.md +++ b/UPGRADE-18.0.md @@ -29,6 +29,7 @@ Please consult the `examples` directory for reference example configurations. If - The previous iteration used a count over a list of node group definitions which was prone to disruptive updates; this is now replaced with a map/for_each to align with that of the EKS managed node group and Fargate profile behaviors/style - The user data configuration supported across the module has been completely revamped. A new `_user_data` internal sub-module has been created to consolidate all user data configuration in one location which provides better support for testability (via the [`examples/user_data`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) example). The new sub-module supports nearly all possible combinations including the ability to allow users to provide their own user data template which will be rendered by the module. See the `examples/user_data` example project for the full plethora of example configuration possibilities and more details on the logic of the design can be found in the [`modules/_user_data`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/_user_data_) directory. - Resource name changes may cause issues with existing resources. For example, security groups and IAM roles cannot be renamed, they must be recreated. Recreation of these resources may also trigger a recreation of the cluster. To use the legacy (< 18.x) resource naming convention, set `prefix_separator` to "". +- Security group usage has been overhauled to provide only the bare minimum network connectivity required to launch a bare bones cluster. See the [security group documentation section](https://github.com/terraform-aws-modules/terraform-aws-eks#security-groups) for more details. Users upgrading to v18.x will want to review the rules they have in place today versus the rules provisioned by the v18.x module and ensure to make any necessary adjustments for their specific workload. ## Additional changes diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index b23c13d3f1..6b662257f9 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -32,6 +32,7 @@ Note that this example may create resources which cost money. Run `terraform des | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.64 | | [null](#requirement\_null) | >= 3.0 | +| [tls](#requirement\_tls) | >= 2.2 | ## Providers @@ -39,6 +40,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|---------| | [aws](#provider\_aws) | >= 3.64 | | [null](#provider\_null) | >= 3.0 | +| [tls](#provider\_tls) | >= 2.2 | ## Modules @@ -51,11 +53,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| +| [aws_key_pair.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | | [aws_kms_key.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_launch_template.external](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [null_resource.patch](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index b45505a198..ce76f94085 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -68,10 +68,25 @@ module "eks" { eks_managed_node_groups = { # Default node group - as provided by AWS EKS - default_node_group = {} + default_node_group = { + # By default, the module creates a launch template to ensure tags are propagated to instances, etc., + # so we need to disable it to use the default template provided by the AWS EKS managed node group service + create_launch_template = false + launch_template_name = "" + + # Remote access cannot be specified with a launch template + remote_access = { + ec2_ssh_key = aws_key_pair.this.key_name + } + } # Default node group - as provided by AWS EKS using Bottlerocket bottlerocket_default = { + # By default, the module creates a launch template to ensure tags are propagated to instances, etc., + # so we need to disable it to use the default template provided by the AWS EKS managed node group service + create_launch_template = false + launch_template_name = "" + ami_type = "BOTTLEROCKET_x86_64" platform = "bottlerocket" } @@ -122,20 +137,23 @@ module "eks" { # Use a custom AMI custom_ami = { + ami_type = "AL2_ARM_64" # Current default AMI used by managed node groups - pseudo "custom" - ami_id = "ami-0caf35bc73450c396" + ami_id = "ami-01dc0aa438e3214c2" # ARM # This will ensure the boostrap user data is used to join the node # By default, EKS managed node groups will not append bootstrap script; # this adds it back in using the default template provided by the module # Note: this assumes the AMI provided is an EKS optimized AMI derivative enable_bootstrap_user_data = true + + instance_types = ["t4g.medium"] } # Complete complete = { name = "complete-eks-mng" - use_name_prefix = false + use_name_prefix = true subnet_ids = module.vpc.private_subnets @@ -173,10 +191,6 @@ module "eks" { } ] - remote_access = { - ec2_ssh_key = "my-ssh-key" - } - update_config = { max_unavailable_percentage = 50 # or set `max_unavailable` } @@ -475,6 +489,7 @@ resource "aws_launch_template" "external" { resource_type = "instance" tags = { + Name = "external_lt" CustomTag = "Instance custom tag" } } @@ -503,3 +518,14 @@ resource "aws_launch_template" "external" { create_before_destroy = true } } + +resource "tls_private_key" "this" { + algorithm = "RSA" +} + +resource "aws_key_pair" "this" { + key_name_prefix = local.name + public_key = tls_private_key.this.public_key_openssh + + tags = local.tags +} diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index adfd0180d4..883963f7b0 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -10,5 +10,9 @@ terraform { source = "hashicorp/null" version = ">= 3.0" } + tls = { + source = "hashicorp/tls" + version = ">= 2.2" + } } } From f54bd3047ba18179766641e347fe9f4fa60ff11b Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 11 Jan 2022 15:03:20 -0500 Subject: [PATCH 440/962] fix: Correct remote access variable for security groups and add example for additional IAM policies (#1766) Co-authored-by: Anton Babenko --- README.md | 33 ++++++++++++ examples/eks_managed_node_group/README.md | 3 ++ examples/eks_managed_node_group/main.tf | 59 ++++++++++++++++++++- modules/eks-managed-node-group/README.md | 2 +- modules/eks-managed-node-group/variables.tf | 2 +- 5 files changed, 96 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 21c134522e..f5ca0031b5 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,39 @@ Terraform module which creates AWS EKS (Kubernetes) resources - Support for providing maps of node groups/Fargate profiles to the cluster module definition or use separate node group/Fargate profile sub-modules - Provisions to provide node group/Fargate profile "default" settings - useful for when creating multiple node groups/Fargate profiles where you want to set a common set of configurations once, and then individual control only select features +### ℹ️ `Error: Invalid for_each argument ...` + +Users may encounter an error such as `Error: Invalid for_each argument - The "for_each" value depends on resource attributes that cannot be determined until apply, so Terraform cannot predict how many instances will be created. To work around this, use the -target argument to first apply ...` + +This error is due to an upstream issue with [Terraform core](https://github.com/hashicorp/terraform/issues/4149). There are two potential options you can take to help mitigate this issue: + +1. Create the dependent resources before the cluster => `terraform apply -target ` and then `terraform apply` for the cluster (or other similar means to just ensure the referenced resources exist before creating the cluster) + - Note: this is the route users will have to take for adding additonal security groups to nodes since there isn't a separate "security group attachment" resource +2. For addtional IAM policies, users can attach the policies outside of the cluster definition as demonstrated below + +```hcl +resource "aws_iam_role_policy_attachment" "additional" { + for_each = module.eks.eks_managed_node_groups + # you could also do the following or any comibination: + # for_each = merge( + # module.eks.eks_managed_node_groups, + # module.eks.self_managed_node_group, + # module.eks.fargate_profile, + # ) + + # This policy does not have to exist at the time of cluster creation. Terraform can + # deduce the proper order of its creation to avoid errors during creation + policy_arn = aws_iam_policy.node_additional.arn + role = each.value.iam_role_name +} +``` + +The tl;dr for this issue is that the Terraform resource passed into the modules map definition *must* be known before you can apply the EKS module. The variables this potentially affects are: + +- `cluster_security_group_additional_rules` (i.e. - referencing an external security group resource in a rule) +- `node_security_group_additional_rules` (i.e. - referencing an external security group resource in a rule) +- `iam_role_additional_policies` (i.e. - referencing an external policy resource) + ## Usage ```hcl diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 6b662257f9..ca475e572f 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -53,11 +53,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| +| [aws_iam_policy.node_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_key_pair.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | | [aws_kms_key.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_launch_template.external](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.remote_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [null_resource.patch](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index ce76f94085..2c540e2ae7 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -76,7 +76,8 @@ module "eks" { # Remote access cannot be specified with a launch template remote_access = { - ec2_ssh_key = aws_key_pair.this.key_name + ec2_ssh_key = aws_key_pair.this.key_name + source_security_group_ids = [aws_security_group.remote_access.id] } } @@ -269,6 +270,18 @@ module "eks" { tags = local.tags } +# References to resources that do not exist yet when creating a cluster will cause a plan failure due to https://github.com/hashicorp/terraform/issues/4149 +# There are two options users can take +# 1. Create the dependent resources before the cluster => `terraform apply -target and then `terraform apply` +# Note: this is the route users will have to take for adding additonal security groups to nodes since there isn't a separate "security group attachment" resource +# 2. For addtional IAM policies, users can attach the policies outside of the cluster definition as demonstrated below +resource "aws_iam_role_policy_attachment" "additional" { + for_each = module.eks.eks_managed_node_groups + + policy_arn = aws_iam_policy.node_additional.arn + role = each.value.iam_role_name +} + ################################################################################ # aws-auth configmap # Only EKS managed node groups automatically add roles to aws-auth configmap @@ -529,3 +542,47 @@ resource "aws_key_pair" "this" { tags = local.tags } + +resource "aws_security_group" "remote_access" { + name_prefix = "${local.name}-remote-access" + description = "Allow remote SSH access" + vpc_id = module.vpc.vpc_id + + ingress { + description = "SSH access" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["10.0.0.0/8"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + tags = local.tags +} + +resource "aws_iam_policy" "node_additional" { + name = "${local.name}-additional" + description = "Example usage of node additional policy" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:Describe*", + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) + + tags = local.tags +} diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 43e4a72bc3..334edb4599 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -139,7 +139,7 @@ module "eks_managed_node_group" { | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | | [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | -| [remote\_access](#input\_remote\_access) | Configuration block with remote access settings | `map(string)` | `{}` | no | +| [remote\_access](#input\_remote\_access) | Configuration block with remote access settings | `any` | `{}` | no | | [security\_group\_description](#input\_security\_group\_description) | Description for the security group created | `string` | `"EKS managed node group security group"` | no | | [security\_group\_name](#input\_security\_group\_name) | Name to use on security group created | `string` | `null` | no | | [security\_group\_rules](#input\_security\_group\_rules) | List of security group rules to add to the security group created | `any` | `{}` | no | diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 38d126f269..d35bcc75d6 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -334,7 +334,7 @@ variable "launch_template_version" { variable "remote_access" { description = "Configuration block with remote access settings" - type = map(string) + type = any default = {} } From 5818de1fbdcca461a917e889e6703218d740494c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 11 Jan 2022 20:03:53 +0000 Subject: [PATCH 441/962] chore(release): version 18.0.6 [skip ci] ### [18.0.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.5...v18.0.6) (2022-01-11) ### Bug Fixes * Correct remote access variable for security groups and add example for additional IAM policies ([#1766](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1766)) ([f54bd30](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f54bd3047ba18179766641e347fe9f4fa60ff11b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f81e8a2700..075d243218 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.0.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.5...v18.0.6) (2022-01-11) + + +### Bug Fixes + +* Correct remote access variable for security groups and add example for additional IAM policies ([#1766](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1766)) ([f54bd30](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f54bd3047ba18179766641e347fe9f4fa60ff11b)) + ### [18.0.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.4...v18.0.5) (2022-01-08) From 78555e1a1383b1dba7a68899666463326d456ea1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 12 Jan 2022 14:23:45 -0500 Subject: [PATCH 442/962] chore: Update user data docs to correct mistake of showing `bootstrap_extra_args` when prepending user data (default AMI) (#1773) --- examples/user_data/main.tf | 15 ++++----------- modules/_user_data/README.md | 1 - modules/_user_data/main.tf | 2 +- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index 4e961a3947..5c5b1266cf 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -19,14 +19,7 @@ module "eks_mng_linux_additional" { source = "../../modules/_user_data" pre_bootstrap_user_data = <<-EOT - echo "foo" - export FOO=bar - EOT - - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" - - post_bootstrap_user_data = <<-EOT - echo "All done" + export CONTAINER_RUNTIME="containerd" EOT } @@ -41,11 +34,11 @@ module "eks_mng_linux_custom_ami" { enable_bootstrap_user_data = true pre_bootstrap_user_data = <<-EOT - echo "foo" - export FOO=bar + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false EOT - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20 --instance-type t3a.large'" post_bootstrap_user_data = <<-EOT echo "All done" diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index f61500517e..51e7b9292e 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -21,7 +21,6 @@ When using an EKS managed node group, users have 2 primary routes for interactin ```hcl pre_bootstrap_user_data = "..." - bootstrap_extra_args = "..." ``` 2. If the EKS managed node group does utilize a custom AMI, then per the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami), users will need to supply the necessary bootstrap configuration via user data to ensure that the node is configured to register with the cluster when launched. There are two routes that users can utilize to facilitate this bootstrapping process: diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index 1d265bdb54..8eec42d36c 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -28,7 +28,7 @@ locals { cluster_endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 # Optional - is appended if using EKS managed node group without custom AMI - # cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr # Not supported yet: https://github.com/bottlerocket-os/bottlerocket/issues/1866 + # cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr # Bottlerocket pulls this automatically https://github.com/bottlerocket-os/bottlerocket/issues/1866 bootstrap_extra_args = var.bootstrap_extra_args } )) : "" From cf985c23471b39d5bf2b7219b338c22e7cdae711 Mon Sep 17 00:00:00 2001 From: yutachaos <18604471+yutachaos@users.noreply.github.com> Date: Thu, 13 Jan 2022 20:28:04 +0900 Subject: [PATCH 443/962] chore: Fixed some broken urls in the docs (#1775) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f5ca0031b5..7f6eea1445 100644 --- a/README.md +++ b/README.md @@ -586,7 +586,7 @@ The security groups created by this module are depicted in the image shown below ## Notes -- Setting `instance_refresh_enabled = true` will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. See the [instance_refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/instance_refresh) example provided. +- Setting `instance_refresh_enabled = true` will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. See the [instance_refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa_autoscale_refresh) example provided.
Frequently Asked Questions
@@ -667,9 +667,9 @@ See your Kubernetes version's documentation for the `--node-labels` kubelet fla - [Complete](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete): EKS Cluster using all available node group types in various combinations demonstrating many of the supported features and configurations - [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group): EKS Cluster using EKS managed node groups - [Fargate Profile](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile): EKS cluster using [Fargate Profiles](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) -- [IRSA, Node Autoscaler, Instance Refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa_autoscale_instance_refresh): EKS Cluster using self-managed node group demonstrating how to enable/utilize instance refresh configuration along with node termination handler +- [IRSA, Node Autoscaler, Instance Refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa_autoscale_refresh): EKS Cluster using self-managed node group demonstrating how to enable/utilize instance refresh configuration along with node termination handler - [Self Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self_managed_node_group): EKS Cluster using self-managed node groups -- [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self_managed_node_group): Various supported methods of providing necessary bootstrap scripts and configuration settings via user data +- [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data): Various supported methods of providing necessary bootstrap scripts and configuration settings via user data ## Contributing From 314192e2ebc5faaf5f027a7d868cd36c4844aee1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 14 Jan 2022 15:17:03 -0500 Subject: [PATCH 444/962] feat: Add support for networking `ip_family` which enables support for IPV6 (#1759) --- README.md | 8 +- examples/complete/README.md | 4 +- examples/complete/versions.tf | 2 +- examples/eks_managed_node_group/README.md | 4 +- examples/eks_managed_node_group/main.tf | 12 ++- examples/eks_managed_node_group/versions.tf | 2 +- examples/fargate_profile/README.md | 4 +- examples/fargate_profile/versions.tf | 2 +- examples/irsa_autoscale_refresh/README.md | 4 +- examples/irsa_autoscale_refresh/versions.tf | 2 +- examples/self_managed_node_group/README.md | 4 +- examples/self_managed_node_group/versions.tf | 2 +- examples/user_data/README.md | 2 +- examples/user_data/versions.tf | 2 +- main.tf | 1 + modules/eks-managed-node-group/README.md | 7 +- modules/eks-managed-node-group/main.tf | 15 ++- modules/eks-managed-node-group/variables.tf | 12 +++ modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 7 +- modules/fargate-profile/main.tf | 12 ++- modules/fargate-profile/variables.tf | 12 +++ modules/fargate-profile/versions.tf | 2 +- modules/self-managed-node-group/README.md | 8 +- modules/self-managed-node-group/main.tf | 6 +- modules/self-managed-node-group/variables.tf | 8 +- modules/self-managed-node-group/versions.tf | 2 +- node_groups.tf | 96 +++++++++++++++----- variables.tf | 16 ++++ versions.tf | 2 +- 30 files changed, 197 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 7f6eea1445..3b4fc2ebf3 100644 --- a/README.md +++ b/README.md @@ -682,14 +682,14 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.64 | +| [aws](#requirement\_aws) | >= 3.72 | | [tls](#requirement\_tls) | >= 2.2 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.64 | +| [aws](#provider\_aws) | >= 3.72 | | [tls](#provider\_tls) | >= 2.2 | ## Modules @@ -709,6 +709,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | | [aws_eks_identity_provider_config.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_identity_provider_config) | resource | | [aws_iam_openid_connect_provider.oidc_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource | +| [aws_iam_policy.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_security_group.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | @@ -716,6 +717,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [aws_security_group_rule.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [tls_certificate.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/data-sources/certificate) | data source | @@ -733,6 +735,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `true` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | | [cluster\_security\_group\_additional\_rules](#input\_cluster\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created | `any` | `{}` | no | | [cluster\_security\_group\_description](#input\_cluster\_security\_group\_description) | Description of the cluster security group created | `string` | `"EKS cluster security group"` | no | @@ -747,6 +750,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | | [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster or use the existing `cluster_security_group_id` | `bool` | `true` | no | +| [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy) | `bool` | `false` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether a an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | diff --git a/examples/complete/README.md b/examples/complete/README.md index e7c9debc8a..6fa1f3e8f3 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -34,14 +34,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.64 | +| [aws](#requirement\_aws) | >= 3.72 | | [null](#requirement\_null) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.64 | +| [aws](#provider\_aws) | >= 3.72 | | [null](#provider\_null) | >= 3.0 | ## Modules diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index adfd0180d4..a53f5a3d28 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.64" + version = ">= 3.72" } null = { source = "hashicorp/null" diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index ca475e572f..3de86a55ef 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -30,7 +30,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.64 | +| [aws](#requirement\_aws) | >= 3.72 | | [null](#requirement\_null) | >= 3.0 | | [tls](#requirement\_tls) | >= 2.2 | @@ -38,7 +38,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.64 | +| [aws](#provider\_aws) | >= 3.72 | | [null](#provider\_null) | >= 3.0 | | [tls](#provider\_tls) | >= 2.2 | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 2c540e2ae7..c4b4a6b9e6 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -25,10 +25,13 @@ module "eks" { cluster_name = local.name cluster_version = local.cluster_version - cluster_service_ipv4_cidr = "172.16.0.0/16" cluster_endpoint_private_access = true cluster_endpoint_public_access = true + # IPV6 + cluster_ip_family = "ipv6" + create_cni_ipv6_iam_policy = true + cluster_addons = { coredns = { resolve_conflicts = "OVERWRITE" @@ -350,6 +353,13 @@ module "vpc" { private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_ipv6 = true + assign_ipv6_address_on_creation = true + create_egress_only_igw = true + + public_subnet_ipv6_prefixes = [0, 1, 2] + private_subnet_ipv6_prefixes = [3, 4, 5] + enable_nat_gateway = true single_nat_gateway = true enable_dns_hostnames = true diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index 883963f7b0..2ba2b42d4e 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.64" + version = ">= 3.72" } null = { source = "hashicorp/null" diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 9ff7b03a4c..85bce85ce7 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -20,13 +20,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.64 | +| [aws](#requirement\_aws) | >= 3.72 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.64 | +| [aws](#provider\_aws) | >= 3.72 | ## Modules diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index bfce6ae345..22e8d7265f 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.64" + version = ">= 3.72" } } } diff --git a/examples/irsa_autoscale_refresh/README.md b/examples/irsa_autoscale_refresh/README.md index a21f31f6d3..d48a2a8edf 100644 --- a/examples/irsa_autoscale_refresh/README.md +++ b/examples/irsa_autoscale_refresh/README.md @@ -24,7 +24,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.64 | +| [aws](#requirement\_aws) | >= 3.72 | | [helm](#requirement\_helm) | >= 2.0 | | [null](#requirement\_null) | >= 3.0 | @@ -32,7 +32,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.64 | +| [aws](#provider\_aws) | >= 3.72 | | [helm](#provider\_helm) | >= 2.0 | | [null](#provider\_null) | >= 3.0 | diff --git a/examples/irsa_autoscale_refresh/versions.tf b/examples/irsa_autoscale_refresh/versions.tf index 4706dec92a..20ac24f966 100644 --- a/examples/irsa_autoscale_refresh/versions.tf +++ b/examples/irsa_autoscale_refresh/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.64" + version = ">= 3.72" } null = { source = "hashicorp/null" diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index ff13d8148a..3dfc50d3ec 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -26,7 +26,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.64 | +| [aws](#requirement\_aws) | >= 3.72 | | [null](#requirement\_null) | >= 3.0 | | [tls](#requirement\_tls) | >= 2.2 | @@ -34,7 +34,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.64 | +| [aws](#provider\_aws) | >= 3.72 | | [null](#provider\_null) | >= 3.0 | | [tls](#provider\_tls) | >= 2.2 | diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index 883963f7b0..2ba2b42d4e 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.64" + version = ">= 3.72" } null = { source = "hashicorp/null" diff --git a/examples/user_data/README.md b/examples/user_data/README.md index 57ba591944..54cd9ec72b 100644 --- a/examples/user_data/README.md +++ b/examples/user_data/README.md @@ -18,7 +18,7 @@ $ terraform apply | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.64 | +| [aws](#requirement\_aws) | >= 3.72 | ## Providers diff --git a/examples/user_data/versions.tf b/examples/user_data/versions.tf index bfce6ae345..22e8d7265f 100644 --- a/examples/user_data/versions.tf +++ b/examples/user_data/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.64" + version = ">= 3.72" } } } diff --git a/main.tf b/main.tf index 3a01c64fef..5797a65649 100644 --- a/main.tf +++ b/main.tf @@ -21,6 +21,7 @@ resource "aws_eks_cluster" "this" { } kubernetes_network_config { + ip_family = var.cluster_ip_family service_ipv4_cidr = var.cluster_service_ipv4_cidr } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 334edb4599..9605e7fef4 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -49,14 +49,14 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.64 | +| [aws](#requirement\_aws) | >= 3.72 | | [cloudinit](#requirement\_cloudinit) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.64 | +| [aws](#provider\_aws) | >= 3.72 | ## Modules @@ -74,6 +74,7 @@ module "eks_managed_node_group" { | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | | [aws_security_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -90,6 +91,7 @@ module "eks_managed_node_group" { | [capacity\_type](#input\_capacity\_type) | Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT` | `string` | `"ON_DEMAND"` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | @@ -112,6 +114,7 @@ module "eks_managed_node_group" { | [force\_update\_version](#input\_force\_update\_version) | Force version update if existing pods are unable to be drained due to a pod disruption budget issue | `bool` | `null` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the node group. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 3188a8d910..32f661c96b 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -1,5 +1,7 @@ data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + ################################################################################ # User Data ################################################################################ @@ -389,8 +391,11 @@ resource "aws_security_group_rule" "this" { ################################################################################ locals { - iam_role_name = coalesce(var.iam_role_name, "${var.name}-eks-node-group") - policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + iam_role_name = coalesce(var.iam_role_name, "${var.name}-eks-node-group") + + iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + + cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } data "aws_iam_policy_document" "assume_role_policy" { @@ -425,9 +430,9 @@ resource "aws_iam_role" "this" { # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group resource "aws_iam_role_policy_attachment" "this" { for_each = var.create && var.create_iam_role ? toset(compact(distinct(concat([ - "${local.policy_arn_prefix}/AmazonEKSWorkerNodePolicy", - "${local.policy_arn_prefix}/AmazonEC2ContainerRegistryReadOnly", - "${local.policy_arn_prefix}/AmazonEKS_CNI_Policy", + "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", + "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", + var.iam_role_attach_cni_policy ? local.cni_policy : "", ], var.iam_role_additional_policies)))) : toset([]) policy_arn = each.value diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index d35bcc75d6..ddefdcf655 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -418,6 +418,12 @@ variable "create_iam_role" { default = true } +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = null +} + variable "iam_role_arn" { description = "Existing IAM role ARN for the node group. Required if `create_iam_role` is set to `false`" type = string @@ -454,6 +460,12 @@ variable "iam_role_permissions_boundary" { default = null } +variable "iam_role_attach_cni_policy" { + description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" + type = bool + default = true +} + variable "iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" type = list(string) diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 48492037e2..e196e178e4 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.64" + version = ">= 3.72" } cloudinit = { source = "hashicorp/cloudinit" diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 8a51f317a8..6851954fdc 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.64 | +| [aws](#requirement\_aws) | >= 3.72 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.64 | +| [aws](#provider\_aws) | >= 3.72 | ## Modules @@ -48,6 +48,7 @@ No modules. | [aws_eks_fargate_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_fargate_profile) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -55,11 +56,13 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `null` | no | | [create](#input\_create) | Determines whether to create Fargate profile or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the Fargate profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `""` | no | | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | diff --git a/modules/fargate-profile/main.tf b/modules/fargate-profile/main.tf index b37fc45116..554b0e8814 100644 --- a/modules/fargate-profile/main.tf +++ b/modules/fargate-profile/main.tf @@ -1,8 +1,13 @@ data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + locals { - iam_role_name = coalesce(var.iam_role_name, var.name, "fargate-profile") - policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + iam_role_name = coalesce(var.iam_role_name, var.name, "fargate-profile") + + iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + + cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } ################################################################################ @@ -40,7 +45,8 @@ resource "aws_iam_role" "this" { resource "aws_iam_role_policy_attachment" "this" { for_each = var.create && var.create_iam_role ? toset(compact(distinct(concat([ - "${local.policy_arn_prefix}/AmazonEKSFargatePodExecutionRolePolicy", + "${local.iam_role_policy_prefix}/AmazonEKSFargatePodExecutionRolePolicy", + var.iam_role_attach_cni_policy ? local.cni_policy : "", ], var.iam_role_additional_policies)))) : toset([]) policy_arn = each.value diff --git a/modules/fargate-profile/variables.tf b/modules/fargate-profile/variables.tf index afad4809a6..5accf35e1b 100644 --- a/modules/fargate-profile/variables.tf +++ b/modules/fargate-profile/variables.tf @@ -20,6 +20,12 @@ variable "create_iam_role" { default = true } +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = null +} + variable "iam_role_arn" { description = "Existing IAM role ARN for the Fargate profile. Required if `create_iam_role` is set to `false`" type = string @@ -56,6 +62,12 @@ variable "iam_role_permissions_boundary" { default = null } +variable "iam_role_attach_cni_policy" { + description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" + type = bool + default = true +} + variable "iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" type = list(string) diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index bfce6ae345..22e8d7265f 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.64" + version = ">= 3.72" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 10a3068825..b1ff6b64fd 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -40,14 +40,14 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.64 | +| [aws](#requirement\_aws) | >= 3.72 | | [cloudinit](#requirement\_cloudinit) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.64 | +| [aws](#provider\_aws) | >= 3.72 | ## Modules @@ -68,6 +68,7 @@ module "self_managed_node_group" { | [aws_security_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -83,6 +84,7 @@ module "self_managed_node_group" { | [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `null` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | @@ -109,7 +111,7 @@ module "self_managed_node_group" { | [hibernation\_options](#input\_hibernation\_options) | The hibernation options for the instance | `map(string)` | `null` | no | | [iam\_instance\_profile\_arn](#input\_iam\_instance\_profile\_arn) | Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false` | `string` | `null` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | -| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | +| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 2acaa3d199..ae303e3c04 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -1,5 +1,7 @@ data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + data "aws_ami" "eks_default" { count = var.create ? 1 : 0 @@ -494,6 +496,8 @@ locals { iam_role_name = coalesce(var.iam_role_name, "${var.name}-node-group") iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + + cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } data "aws_iam_policy_document" "assume_role_policy" { @@ -529,7 +533,7 @@ resource "aws_iam_role_policy_attachment" "this" { for_each = var.create && var.create_iam_instance_profile ? toset(compact(distinct(concat([ "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", - var.iam_role_attach_cni_policy ? "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" : "", + var.iam_role_attach_cni_policy ? local.cni_policy : "", ], var.iam_role_additional_policies)))) : toset([]) policy_arn = each.value diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index d5fe48dec5..7bcebeaddc 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -524,6 +524,12 @@ variable "create_iam_instance_profile" { default = true } +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = null +} + variable "iam_instance_profile_arn" { description = "Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false`" type = string @@ -561,7 +567,7 @@ variable "iam_role_permissions_boundary" { } variable "iam_role_attach_cni_policy" { - description = "Whether to attach the Amazon managed `AmazonEKS_CNI_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" + description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" type = bool default = true } diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 48492037e2..e196e178e4 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.64" + version = ">= 3.72" } cloudinit = { source = "hashicorp/cloudinit" diff --git a/node_groups.tf b/node_groups.tf index 18bd23a92f..0f98647b6e 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -6,6 +6,46 @@ locals { } } +################################################################################ +# EKS IPV6 CNI Policy +# TODO - hopefully AWS releases a managed policy which can replace this +# https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy +################################################################################ + +data "aws_iam_policy_document" "cni_ipv6_policy" { + count = var.create && var.create_cni_ipv6_iam_policy ? 1 : 0 + + statement { + sid = "AssignDescribe" + actions = [ + "ec2:AssignIpv6Addresses", + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeInstanceTypes" + ] + resources = ["*"] + } + + statement { + sid = "CreateTags" + actions = ["ec2:CreateTags"] + resources = ["arn:aws:ec2:*:*:network-interface/*"] + } +} + +# Note - we are keeping this to a minimim in hopes that its soon replaced with an AWS managed policy like `AmazonEKS_CNI_Policy` +resource "aws_iam_policy" "cni_ipv6_policy" { + count = var.create && var.create_cni_ipv6_iam_policy ? 1 : 0 + + # Will cause conflicts if trying to create on multiple clusters but necessary to reference by exact name in sub-modules + name = "AmazonEKS_CNI_IPv6_Policy" + description = "IAM policy for EKS CNI to assign IPV6 addresses" + policy = data.aws_iam_policy_document.cni_ipv6_policy[0].json + + tags = var.tags +} + ################################################################################ # Node Security Group # Defaults follow https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html @@ -76,28 +116,31 @@ locals { self = true } egress_https = { - description = "Egress all HTTPS to internet" - protocol = "tcp" - from_port = 443 - to_port = 443 - type = "egress" - cidr_blocks = ["0.0.0.0/0"] + description = "Egress all HTTPS to internet" + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null } egress_ntp_tcp = { - description = "Egress NTP/TCP to internet" - protocol = "tcp" - from_port = 123 - to_port = 123 - type = "egress" - cidr_blocks = ["0.0.0.0/0"] + description = "Egress NTP/TCP to internet" + protocol = "tcp" + from_port = 123 + to_port = 123 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null } egress_ntp_udp = { - description = "Egress NTP/UDP to internet" - protocol = "udp" - from_port = 123 - to_port = 123 - type = "egress" - cidr_blocks = ["0.0.0.0/0"] + description = "Egress NTP/UDP to internet" + protocol = "udp" + from_port = 123 + to_port = 123 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null } } } @@ -152,11 +195,12 @@ module "fargate_profile" { for_each = { for k, v in var.fargate_profiles : k => v if var.create } # Fargate Profile - cluster_name = aws_eks_cluster.this[0].name - name = try(each.value.name, each.key) - subnet_ids = try(each.value.subnet_ids, var.fargate_profile_defaults.subnet_ids, var.subnet_ids) - selectors = try(each.value.selectors, var.fargate_profile_defaults.selectors, []) - timeouts = try(each.value.timeouts, var.fargate_profile_defaults.timeouts, {}) + cluster_name = aws_eks_cluster.this[0].name + cluster_ip_family = var.cluster_ip_family + name = try(each.value.name, each.key) + subnet_ids = try(each.value.subnet_ids, var.fargate_profile_defaults.subnet_ids, var.subnet_ids) + selectors = try(each.value.selectors, var.fargate_profile_defaults.selectors, []) + timeouts = try(each.value.timeouts, var.fargate_profile_defaults.timeouts, {}) # IAM role create_iam_role = try(each.value.create_iam_role, var.fargate_profile_defaults.create_iam_role, true) @@ -167,6 +211,7 @@ module "fargate_profile" { iam_role_description = try(each.value.iam_role_description, var.fargate_profile_defaults.iam_role_description, "Fargate profile IAM role") iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.fargate_profile_defaults.iam_role_permissions_boundary, null) iam_role_tags = try(each.value.iam_role_tags, var.fargate_profile_defaults.iam_role_tags, {}) + iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.fargate_profile_defaults.iam_role_attach_cni_policy, true) iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.fargate_profile_defaults.iam_role_additional_policies, []) tags = merge(var.tags, try(each.value.tags, var.fargate_profile_defaults.tags, {})) @@ -184,6 +229,7 @@ module "eks_managed_node_group" { cluster_name = aws_eks_cluster.this[0].name cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, var.cluster_version) cluster_security_group_id = local.cluster_security_group_id + cluster_ip_family = var.cluster_ip_family # EKS Managed Node Group name = try(each.value.name, each.key) @@ -260,6 +306,7 @@ module "eks_managed_node_group" { iam_role_description = try(each.value.iam_role_description, var.eks_managed_node_group_defaults.iam_role_description, "EKS managed node group IAM role") iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.eks_managed_node_group_defaults.iam_role_permissions_boundary, null) iam_role_tags = try(each.value.iam_role_tags, var.eks_managed_node_group_defaults.iam_role_tags, {}) + iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.eks_managed_node_group_defaults.iam_role_attach_cni_policy, true) iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.eks_managed_node_group_defaults.iam_role_additional_policies, []) # Security group @@ -283,7 +330,8 @@ module "self_managed_node_group" { for_each = { for k, v in var.self_managed_node_groups : k => v if var.create } - cluster_name = aws_eks_cluster.this[0].name + cluster_name = aws_eks_cluster.this[0].name + cluster_ip_family = var.cluster_ip_family # Autoscaling Group name = try(each.value.name, each.key) diff --git a/variables.tf b/variables.tf index 5e715b1b8c..5452649421 100644 --- a/variables.tf +++ b/variables.tf @@ -68,6 +68,12 @@ variable "cluster_endpoint_public_access_cidrs" { default = ["0.0.0.0/0"] } +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created" + type = string + default = null +} + variable "cluster_service_ipv4_cidr" { description = "The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" type = string @@ -169,6 +175,16 @@ variable "cluster_security_group_tags" { default = {} } +################################################################################ +# EKS IPV6 CNI Policy +################################################################################ + +variable "create_cni_ipv6_iam_policy" { + description = "Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy)" + type = bool + default = false +} + ################################################################################ # Node Security Group ################################################################################ diff --git a/versions.tf b/versions.tf index 1969855a84..cb7e557e9d 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.64" + version = ">= 3.72" } tls = { source = "hashicorp/tls" From aba94002e419e0805891ba4e4b5cbd12db7a1e9e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 14 Jan 2022 20:17:35 +0000 Subject: [PATCH 445/962] chore(release): version 18.1.0 [skip ci] ## [18.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.6...v18.1.0) (2022-01-14) ### Features * Add support for networking `ip_family` which enables support for IPV6 ([#1759](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1759)) ([314192e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/314192e2ebc5faaf5f027a7d868cd36c4844aee1)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 075d243218..de88354bbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.6...v18.1.0) (2022-01-14) + + +### Features + +* Add support for networking `ip_family` which enables support for IPV6 ([#1759](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1759)) ([314192e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/314192e2ebc5faaf5f027a7d868cd36c4844aee1)) + ### [18.0.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.5...v18.0.6) (2022-01-11) From 85bb1a00b6111845141a8c07a9459bbd160d7ed3 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 14 Jan 2022 15:22:58 -0500 Subject: [PATCH 446/962] feat: Add `instance_metadata_tags` attribute to launch templates (#1781) --- examples/eks_managed_node_group/main.tf | 1 + examples/self_managed_node_group/main.tf | 1 + modules/eks-managed-node-group/main.tf | 1 + modules/self-managed-node-group/main.tf | 1 + node_groups.tf | 1 + 5 files changed, 5 insertions(+) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index c4b4a6b9e6..33df785214 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -225,6 +225,7 @@ module "eks" { http_endpoint = "enabled" http_tokens = "required" http_put_response_hop_limit = 2 + instance_metadata_tags = "enabled" } create_iam_role = true diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 73b502520c..59234fe12b 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -138,6 +138,7 @@ module "eks" { http_endpoint = "enabled" http_tokens = "required" http_put_response_hop_limit = 2 + instance_metadata_tags = "enabled" } create_iam_role = true diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 32f661c96b..bbe02105d2 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -183,6 +183,7 @@ resource "aws_launch_template" "this" { http_tokens = lookup(metadata_options.value, "http_tokens", null) http_put_response_hop_limit = lookup(metadata_options.value, "http_put_response_hop_limit", null) http_protocol_ipv6 = lookup(metadata_options.value, "http_protocol_ipv6", null) + instance_metadata_tags = lookup(metadata_options.value, "instance_metadata_tags", null) } } diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index ae303e3c04..8750892e3c 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -182,6 +182,7 @@ resource "aws_launch_template" "this" { http_tokens = lookup(metadata_options.value, "http_tokens", null) http_put_response_hop_limit = lookup(metadata_options.value, "http_put_response_hop_limit", null) http_protocol_ipv6 = lookup(metadata_options.value, "http_protocol_ipv6", null) + instance_metadata_tags = lookup(metadata_options.value, "instance_metadata_tags", null) } } diff --git a/node_groups.tf b/node_groups.tf index 0f98647b6e..fb59e76dc9 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -3,6 +3,7 @@ locals { http_endpoint = "enabled" http_tokens = "required" http_put_response_hop_limit = 2 + instance_metadata_tags = "enabled" } } From 9c6c36c5c8d6071055b2ee190ab7c1685daf6ff0 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 14 Jan 2022 20:23:36 +0000 Subject: [PATCH 447/962] chore(release): version 18.2.0 [skip ci] ## [18.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.1.0...v18.2.0) (2022-01-14) ### Features * Add `instance_metadata_tags` attribute to launch templates ([#1781](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1781)) ([85bb1a0](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/85bb1a00b6111845141a8c07a9459bbd160d7ed3)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de88354bbc..1be0505fe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.1.0...v18.2.0) (2022-01-14) + + +### Features + +* Add `instance_metadata_tags` attribute to launch templates ([#1781](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1781)) ([85bb1a0](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/85bb1a00b6111845141a8c07a9459bbd160d7ed3)) + ## [18.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.6...v18.1.0) (2022-01-14) From fcd4abb5708fa10569e407f028e1dee871c1d3c8 Mon Sep 17 00:00:00 2001 From: Nick Janetakis Date: Mon, 17 Jan 2022 12:17:33 -0500 Subject: [PATCH 448/962] docs: Add use case examples to the 18.x upgrade guide (#1783) --- UPGRADE-18.0.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/UPGRADE-18.0.md b/UPGRADE-18.0.md index 8c1bd66983..2d00e6fa98 100644 --- a/UPGRADE-18.0.md +++ b/UPGRADE-18.0.md @@ -551,3 +551,36 @@ module "cluster_after" { } } ``` + +### Attaching an IAM role policy to a Fargate profile + +#### Before 17.x + +```hcl +resource "aws_iam_role_policy_attachment" "default" { + role = module.eks.fargate_iam_role_name + policy_arn = aws_iam_policy.default.arn +} +``` + +#### After 18.x + +```hcl +# Attach the policy to an "example" Fargate profile +resource "aws_iam_role_policy_attachment" "default" { + role = module.eks.fargate_profiles["example"].iam_role_name + policy_arn = aws_iam_policy.default.arn +} +``` + +Or: + +```hcl +# Attach the policy to all Fargate profiles +resource "aws_iam_role_policy_attachment" "default" { + for_each = module.eks.fargate_profiles + + role = each.value.iam_role_name + policy_arn = aws_iam_policy.default.arn +} +``` From 8e4dfa2be5c60e98a9b20a8ae716c5c446fe935c Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 18 Jan 2022 09:03:09 -0500 Subject: [PATCH 449/962] fix: Change `instance_metadata_tags` to default to `null`/`disabled` due to tag key pattern conflict (#1788) --- examples/eks_managed_node_group/main.tf | 2 +- examples/self_managed_node_group/main.tf | 2 +- node_groups.tf | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 33df785214..13c8bdfc4f 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -225,7 +225,7 @@ module "eks" { http_endpoint = "enabled" http_tokens = "required" http_put_response_hop_limit = 2 - instance_metadata_tags = "enabled" + instance_metadata_tags = "disabled" } create_iam_role = true diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 59234fe12b..6163fcc34b 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -138,7 +138,7 @@ module "eks" { http_endpoint = "enabled" http_tokens = "required" http_put_response_hop_limit = 2 - instance_metadata_tags = "enabled" + instance_metadata_tags = "disabled" } create_iam_role = true diff --git a/node_groups.tf b/node_groups.tf index fb59e76dc9..0f98647b6e 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -3,7 +3,6 @@ locals { http_endpoint = "enabled" http_tokens = "required" http_put_response_hop_limit = 2 - instance_metadata_tags = "enabled" } } From 9f85dc8cf5028ed2bab49f495f58e68aa870e7d4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 18 Jan 2022 14:03:43 +0000 Subject: [PATCH 450/962] chore(release): version 18.2.1 [skip ci] ### [18.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.0...v18.2.1) (2022-01-18) ### Bug Fixes * Change `instance_metadata_tags` to default to `null`/`disabled` due to tag key pattern conflict ([#1788](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1788)) ([8e4dfa2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8e4dfa2be5c60e98a9b20a8ae716c5c446fe935c)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1be0505fe2..4d7d704fe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.0...v18.2.1) (2022-01-18) + + +### Bug Fixes + +* Change `instance_metadata_tags` to default to `null`/`disabled` due to tag key pattern conflict ([#1788](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1788)) ([8e4dfa2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8e4dfa2be5c60e98a9b20a8ae716c5c446fe935c)) + ## [18.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.1.0...v18.2.0) (2022-01-14) From f8fe584d5b50cc4009ac6c34e3bbb33a4e282f2e Mon Sep 17 00:00:00 2001 From: Vincent Yin Date: Sat, 22 Jan 2022 02:15:38 -0500 Subject: [PATCH 451/962] fix: Attributes in timeouts are erroneously reversed (#1804) --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 5797a65649..e7f1f113b8 100644 --- a/main.tf +++ b/main.tf @@ -43,8 +43,8 @@ resource "aws_eks_cluster" "this" { timeouts { create = lookup(var.cluster_timeouts, "create", null) - delete = lookup(var.cluster_timeouts, "update", null) - update = lookup(var.cluster_timeouts, "delete", null) + update = lookup(var.cluster_timeouts, "update", null) + delete = lookup(var.cluster_timeouts, "delete", null) } depends_on = [ From 243d93adff6fe77838cbdd8fd8d217b04123843f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 22 Jan 2022 07:16:21 +0000 Subject: [PATCH 452/962] chore(release): version 18.2.2 [skip ci] ### [18.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.1...v18.2.2) (2022-01-22) ### Bug Fixes * Attributes in timeouts are erroneously reversed ([#1804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1804)) ([f8fe584](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f8fe584d5b50cc4009ac6c34e3bbb33a4e282f2e)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d7d704fe6..8334be49ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.1...v18.2.2) (2022-01-22) + + +### Bug Fixes + +* Attributes in timeouts are erroneously reversed ([#1804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1804)) ([f8fe584](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f8fe584d5b50cc4009ac6c34e3bbb33a4e282f2e)) + ### [18.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.0...v18.2.1) (2022-01-18) From c45adce32e4a109f18c7069b5415e91679deab16 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 22 Jan 2022 02:17:09 -0500 Subject: [PATCH 453/962] chore: add example usage of extending cluster and node security group access rules (#1805) --- README.md | 38 ++++++++++++++++++++ examples/complete/main.tf | 33 ++++++++++++++++++ examples/eks_managed_node_group/main.tf | 44 ++++++++++++++++++------ examples/self_managed_node_group/main.tf | 33 ++++++++++++++++++ 4 files changed, 137 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 3b4fc2ebf3..17d4a325db 100644 --- a/README.md +++ b/README.md @@ -578,6 +578,44 @@ Module provided default templates can be found under the [templates directory](h - Each node group (EKS Managed Node Group and Self Managed Node Group) by default creates its own security group. By default, this security group does not contain any additional security group rules. It is merely an "empty container" that offers users the ability to opt into any addition inbound our outbound rules as necessary - Users also have the option to supply their own, and/or additional, externally created security group(s) to the node group as well via the `vpc_security_group_ids` variable +See the example snippet below which adds additional security group rules to the cluster security group as well as the shared node security group (for node-to-node access). Users can use this extensibility to open up network access as they see fit using the security groups provided by the module: + +```hcl + ... + # Extend cluster security group rules + cluster_security_group_additional_rules = { + egress_nodes_ephemeral_ports_tcp = { + description = "To node 1025-65535" + protocol = "tcp" + from_port = 1025 + to_port = 65535 + type = "egress" + source_node_security_group = true + } + } + + # Extend node-to-node security group rules + node_security_group_additional_rules = { + ingress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + egress_all = { + description = "Node all egress" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + } + ... +``` The security groups created by this module are depicted in the image shown below along with their default inbound/outbound rules:

diff --git a/examples/complete/main.tf b/examples/complete/main.tf index ff32473737..920facdcc3 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -46,6 +46,39 @@ module "eks" { enable_irsa = true + # Extend cluster security group rules + cluster_security_group_additional_rules = { + egress_nodes_ephemeral_ports_tcp = { + description = "To node 1025-65535" + protocol = "tcp" + from_port = 1025 + to_port = 65535 + type = "egress" + source_node_security_group = true + } + } + + # Extend node-to-node security group rules + node_security_group_additional_rules = { + ingress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + egress_all = { + description = "Node all egress" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + } + # Self Managed Node Group(s) self_managed_node_group_defaults = { vpc_security_group_ids = [aws_security_group.additional.id] diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 13c8bdfc4f..f95f5d654e 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -47,22 +47,44 @@ module "eks" { resources = ["secrets"] }] - cluster_security_group_additional_rules = { - admin_access = { - description = "Admin ingress to Kubernetes API" - cidr_blocks = ["10.97.0.0/30"] - protocol = "tcp" - from_port = 443 - to_port = 443 - type = "ingress" - } - } - vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets enable_irsa = true + # Extend cluster security group rules + cluster_security_group_additional_rules = { + egress_nodes_ephemeral_ports_tcp = { + description = "To node 1025-65535" + protocol = "tcp" + from_port = 1025 + to_port = 65535 + type = "egress" + source_node_security_group = true + } + } + + # Extend node-to-node security group rules + node_security_group_additional_rules = { + ingress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + egress_all = { + description = "Node all egress" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + } + eks_managed_node_group_defaults = { ami_type = "AL2_x86_64" disk_size = 50 diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 6163fcc34b..1c77c599c8 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -48,6 +48,39 @@ module "eks" { enable_irsa = true + # Extend cluster security group rules + cluster_security_group_additional_rules = { + egress_nodes_ephemeral_ports_tcp = { + description = "To node 1025-65535" + protocol = "tcp" + from_port = 1025 + to_port = 65535 + type = "egress" + source_node_security_group = true + } + } + + # Extend node-to-node security group rules + node_security_group_additional_rules = { + ingress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + egress_all = { + description = "Node all egress" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + } + self_managed_node_group_defaults = { disk_size = 50 } From 4af77f244a558ec66db6561488a5d8cd0c0f1aed Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 24 Jan 2022 15:19:13 -0500 Subject: [PATCH 454/962] fix: Add missing `mixed_instances_policy` parameter to the root module (#1808) --- examples/self_managed_node_group/main.tf | 30 ++++++++++++++++++++++++ node_groups.tf | 1 + 2 files changed, 31 insertions(+) diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 1c77c599c8..90d97d5a45 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -118,6 +118,36 @@ module "eks" { EOT } + mixed = { + name = "mixed" + + min_size = 1 + max_size = 5 + desired_size = 2 + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + use_mixed_instances_policy = true + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 20 + spot_allocation_strategy = "capacity-optimized" + } + + override = [ + { + instance_type = "m5.large" + weighted_capacity = "1" + }, + { + instance_type = "m6i.large" + weighted_capacity = "2" + }, + ] + } + } + # Complete complete = { name = "complete-self-mng" diff --git a/node_groups.tf b/node_groups.tf index 0f98647b6e..02eaba1be0 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -367,6 +367,7 @@ module "self_managed_node_group" { initial_lifecycle_hooks = try(each.value.initial_lifecycle_hooks, var.self_managed_node_group_defaults.initial_lifecycle_hooks, []) instance_refresh = try(each.value.instance_refresh, var.self_managed_node_group_defaults.instance_refresh, null) use_mixed_instances_policy = try(each.value.use_mixed_instances_policy, var.self_managed_node_group_defaults.use_mixed_instances_policy, false) + mixed_instances_policy = try(each.value.mixed_instances_policy, var.self_managed_node_group_defaults.mixed_instances_policy, null) warm_pool = try(each.value.warm_pool, var.self_managed_node_group_defaults.warm_pool, null) create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, false) From 16f46db94b7158fd762d9133119206aaa7cf6d63 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 24 Jan 2022 20:19:53 +0000 Subject: [PATCH 455/962] chore(release): version 18.2.3 [skip ci] ### [18.2.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.2...v18.2.3) (2022-01-24) ### Bug Fixes * Add missing `mixed_instances_policy` parameter to the root module ([#1808](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1808)) ([4af77f2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/4af77f244a558ec66db6561488a5d8cd0c0f1aed)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8334be49ff..6ff8bedd2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.2.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.2...v18.2.3) (2022-01-24) + + +### Bug Fixes + +* Add missing `mixed_instances_policy` parameter to the root module ([#1808](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1808)) ([4af77f2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/4af77f244a558ec66db6561488a5d8cd0c0f1aed)) + ### [18.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.1...v18.2.2) (2022-01-22) From d6888b5eb6748a065063b0679f228f9fbbf93284 Mon Sep 17 00:00:00 2001 From: efaif <84377713+efaif@users.noreply.github.com> Date: Sun, 30 Jan 2022 19:15:21 +0200 Subject: [PATCH 456/962] fix: add missing `launch_template_use_name_prefix` parameter to the root module (#1818) --- node_groups.tf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/node_groups.tf b/node_groups.tf index 02eaba1be0..a0884356f0 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -385,10 +385,11 @@ module "self_managed_node_group" { user_data_template_path = try(each.value.user_data_template_path, var.self_managed_node_group_defaults.user_data_template_path, "") # Launch Template - create_launch_template = try(each.value.create_launch_template, var.self_managed_node_group_defaults.create_launch_template, true) - launch_template_name = try(each.value.launch_template_name, each.key) - launch_template_version = try(each.value.launch_template_version, var.self_managed_node_group_defaults.launch_template_version, null) - launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") + create_launch_template = try(each.value.create_launch_template, var.self_managed_node_group_defaults.create_launch_template, true) + launch_template_name = try(each.value.launch_template_name, each.key) + launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.self_managed_node_group_defaults.launch_template_use_name_prefix, true) + launch_template_version = try(each.value.launch_template_version, var.self_managed_node_group_defaults.launch_template_version, null) + launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") From f9403cf8b26ad0c109c615a025c86795ea5d301f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 30 Jan 2022 17:15:55 +0000 Subject: [PATCH 457/962] chore(release): version 18.2.4 [skip ci] ### [18.2.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.3...v18.2.4) (2022-01-30) ### Bug Fixes * add missing `launch_template_use_name_prefix` parameter to the root module ([#1818](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1818)) ([d6888b5](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d6888b5eb6748a065063b0679f228f9fbbf93284)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ff8bedd2e..faa1c1cb89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.2.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.3...v18.2.4) (2022-01-30) + + +### Bug Fixes + +* add missing `launch_template_use_name_prefix` parameter to the root module ([#1818](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1818)) ([d6888b5](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d6888b5eb6748a065063b0679f228f9fbbf93284)) + ### [18.2.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.2...v18.2.3) (2022-01-24) From e16b3c4cbd5f139d54467965f690e79f8e68b76b Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 1 Feb 2022 12:44:43 -0500 Subject: [PATCH 458/962] fix: Correct issue where custom launch template is not used when EKS managed node group is used externally (#1824) --- examples/eks_managed_node_group/README.md | 30 +++++++++++ examples/eks_managed_node_group/main.tf | 63 +++++++++++++++++++--- examples/self_managed_node_group/README.md | 3 +- examples/self_managed_node_group/main.tf | 22 +++++--- modules/eks-managed-node-group/main.tf | 10 +++- 5 files changed, 112 insertions(+), 16 deletions(-) diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 3de86a55ef..dad516ec15 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -12,6 +12,33 @@ Configuration in this directory creates an AWS EKS cluster with various EKS Mana See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for further details. +## Container Runtime & User Data + +When using the default AMI provided by the EKS Managed Node Group service (i.e. - not specifying a value for `ami_id`), users should be aware of the limitations of configuring the node bootstrap process via user data. Due to not having direct access to the bootrap.sh script invocation and therefore its configuration flags (this is provide by the EKS Managed Node Group service in the node user data), a work around for ensuring the appropriate configuration settings is shown below. The following example shows how to inject configuration variables ahead of the merged user data provided by the EKS Managed Node Group service as well as how to enable the containerd runtime using this approach. More details can be found [here](https://github.com/awslabs/amazon-eks-ami/issues/844). + +```hcl + ... + # Demo of containerd usage when not specifying a custom AMI ID + # (merged into user data before EKS MNG provided user data) + containerd = { + name = "containerd" + + # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 + pre_bootstrap_user_data = <<-EOT + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + export KUBELET_EXTRA_ARGS="--max-pods=110" + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + EOT + } + ... +``` + ## Usage To run this example you need to execute: @@ -63,6 +90,9 @@ Note that this example may create resources which cost money. Run `terraform des | [aws_security_group.remote_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [null_resource.patch](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | +| [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_ami.eks_default_arm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index f95f5d654e..2739f2afcb 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -133,7 +133,7 @@ module "eks" { # Custom AMI, using module provided bootstrap data bottlerocket_custom = { # Current bottlerocket AMI - ami_id = "ami-0ff61e0bcfc81dc94" + ami_id = data.aws_ami.eks_default_bottlerocket.image_id platform = "bottlerocket" # use module user data template to boostrap @@ -165,7 +165,7 @@ module "eks" { custom_ami = { ami_type = "AL2_ARM_64" # Current default AMI used by managed node groups - pseudo "custom" - ami_id = "ami-01dc0aa438e3214c2" # ARM + ami_id = data.aws_ami.eks_default_arm.image_id # This will ensure the boostrap user data is used to join the node # By default, EKS managed node groups will not append bootstrap script; @@ -176,6 +176,25 @@ module "eks" { instance_types = ["t4g.medium"] } + # Demo of containerd usage when not specifying a custom AMI ID + # (merged into user data before EKS MNG provided user data) + containerd = { + name = "containerd" + + # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 + pre_bootstrap_user_data = <<-EOT + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + export KUBELET_EXTRA_ARGS="--max-pods=110" + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + EOT + } + # Complete complete = { name = "complete-eks-mng" @@ -187,23 +206,23 @@ module "eks" { max_size = 7 desired_size = 1 - ami_id = "ami-0caf35bc73450c396" + ami_id = data.aws_ami.eks_default.image_id enable_bootstrap_user_data = true bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false EOT post_bootstrap_user_data = <<-EOT - echo "you are free little kubelet!" + echo "you are free little kubelet!" EOT capacity_type = "SPOT" disk_size = 256 force_update_version = true - instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large", "m3.large", "m4.large"] + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] labels = { GithubRepo = "terraform-aws-eks" GithubOrg = "terraform-aws-modules" @@ -619,3 +638,33 @@ resource "aws_iam_policy" "node_additional" { tags = local.tags } + +data "aws_ami" "eks_default" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["amazon-eks-node-${local.cluster_version}-v*"] + } +} + +data "aws_ami" "eks_default_arm" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["amazon-eks-arm64-node-${local.cluster_version}-v*"] + } +} + +data "aws_ami" "eks_default_bottlerocket" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"] + } +} diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 3dfc50d3ec..8356943906 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -55,7 +55,8 @@ Note that this example may create resources which cost money. Run `terraform des | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [null_resource.apply](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | -| [aws_ami.bottlerocket_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 90d97d5a45..b8598d7065 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -94,7 +94,7 @@ module "eks" { name = "bottlerocket-self-mng" platform = "bottlerocket" - ami_id = data.aws_ami.bottlerocket_ami.id + ami_id = data.aws_ami.eks_default_bottlerocket.id instance_type = "m5.large" desired_size = 2 key_name = aws_key_pair.this.key_name @@ -159,16 +159,16 @@ module "eks" { max_size = 7 desired_size = 1 - ami_id = "ami-0caf35bc73450c396" + ami_id = data.aws_ami.eks_default.id bootstrap_extra_args = "--kubelet-extra-args '--max-pods=110'" pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false EOT post_bootstrap_user_data = <<-EOT - echo "you are free little kubelet!" + echo "you are free little kubelet!" EOT disk_size = 256 @@ -374,7 +374,17 @@ resource "aws_kms_key" "eks" { tags = local.tags } -data "aws_ami" "bottlerocket_ami" { +data "aws_ami" "eks_default" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["amazon-eks-node-${local.cluster_version}-v*"] + } +} + +data "aws_ami" "eks_default_bottlerocket" { most_recent = true owners = ["amazon"] diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index bbe02105d2..71972b22e9 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -30,8 +30,14 @@ module "user_data" { ################################################################################ locals { - use_custom_launch_template = var.launch_template_name != "" - launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-eks-node-group") + # There are 4 scenarios here that have to be considered for `use_custom_launch_template`: + # 1. `var.create_launch_template = false && var.launch_template_name == ""` => EKS MNG will use its own default LT + # 2. `var.create_launch_template = false && var.launch_template_name == "something"` => User provided custom LT will be used + # 3. `var.create_launch_template = true && var.launch_template_name == ""` => Custom LT will be used, module will provide a default name + # 4. `var.create_launch_template = true && var.launch_template_name == "something"` => Custom LT will be used, LT name is provided by user + use_custom_launch_template = var.create_launch_template || var.launch_template_name != "" + + launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-eks-node-group") } resource "aws_launch_template" "this" { From b38124bd5435a853792cd0abb54c2a3ea11cca49 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 1 Feb 2022 17:45:16 +0000 Subject: [PATCH 459/962] chore(release): version 18.2.5 [skip ci] ### [18.2.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.4...v18.2.5) (2022-02-01) ### Bug Fixes * Correct issue where custom launch template is not used when EKS managed node group is used externally ([#1824](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1824)) ([e16b3c4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e16b3c4cbd5f139d54467965f690e79f8e68b76b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index faa1c1cb89..193d982056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.2.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.4...v18.2.5) (2022-02-01) + + +### Bug Fixes + +* Correct issue where custom launch template is not used when EKS managed node group is used externally ([#1824](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1824)) ([e16b3c4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e16b3c4cbd5f139d54467965f690e79f8e68b76b)) + ### [18.2.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.3...v18.2.4) (2022-01-30) From 776009d74b16e97974534668ca01a950d660166a Mon Sep 17 00:00:00 2001 From: Kaji Date: Wed, 2 Feb 2022 03:21:51 +0900 Subject: [PATCH 460/962] fix: Wrong rolearn in aws_auth_configmap_yaml (#1820) --- examples/complete/main.tf | 2 +- modules/fargate-profile/README.md | 1 + modules/fargate-profile/outputs.tf | 5 +++++ outputs.tf | 8 ++++---- templates/aws_auth_cm.tpl | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 920facdcc3..67777884d2 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -309,7 +309,7 @@ locals { groups: - system:bootstrappers - system:nodes - - rolearn: ${module.fargate_profile.fargate_profile_arn} + - rolearn: ${module.fargate_profile.fargate_profile_pod_execution_role_arn} username: system:node:{{SessionName}} groups: - system:bootstrappers diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 6851954fdc..42349dd2a3 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -81,6 +81,7 @@ No modules. |------|-------------| | [fargate\_profile\_arn](#output\_fargate\_profile\_arn) | Amazon Resource Name (ARN) of the EKS Fargate Profile | | [fargate\_profile\_id](#output\_fargate\_profile\_id) | EKS Cluster name and EKS Fargate Profile name separated by a colon (`:`) | +| [fargate\_profile\_pod\_execution\_role\_arn](#output\_fargate\_profile\_pod\_execution\_role\_arn) | Amazon Resource Name (ARN) of the EKS Fargate Profile Pod execution role ARN | | [fargate\_profile\_status](#output\_fargate\_profile\_status) | Status of the EKS Fargate Profile | | [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | | [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | diff --git a/modules/fargate-profile/outputs.tf b/modules/fargate-profile/outputs.tf index bb6e023248..0bf2edd180 100644 --- a/modules/fargate-profile/outputs.tf +++ b/modules/fargate-profile/outputs.tf @@ -35,3 +35,8 @@ output "fargate_profile_status" { description = "Status of the EKS Fargate Profile" value = try(aws_eks_fargate_profile.this[0].status, "") } + +output "fargate_profile_pod_execution_role_arn" { + description = "Amazon Resource Name (ARN) of the EKS Fargate Profile Pod execution role ARN" + value = try(aws_eks_fargate_profile.this[0].pod_execution_role_arn, "") +} diff --git a/outputs.tf b/outputs.tf index 72ba73d676..36799b2dc5 100644 --- a/outputs.tf +++ b/outputs.tf @@ -165,10 +165,10 @@ output "aws_auth_configmap_yaml" { description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" value = templatefile("${path.module}/templates/aws_auth_cm.tpl", { - eks_managed_role_arns = [for group in module.eks_managed_node_group : group.iam_role_arn] - self_managed_role_arns = [for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"] - win32_self_managed_role_arns = [for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"] - fargate_profile_arns = [for group in module.fargate_profile : group.fargate_profile_arn] + eks_managed_role_arns = [for group in module.eks_managed_node_group : group.iam_role_arn] + self_managed_role_arns = [for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"] + win32_self_managed_role_arns = [for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"] + fargate_profile_pod_execution_role_arns = [for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn] } ) } diff --git a/templates/aws_auth_cm.tpl b/templates/aws_auth_cm.tpl index abf2102a54..73a898e966 100644 --- a/templates/aws_auth_cm.tpl +++ b/templates/aws_auth_cm.tpl @@ -27,7 +27,7 @@ data: - system:bootstrappers - system:nodes %{ endfor ~} -%{ for role in fargate_profile_arns ~} +%{ for role in fargate_profile_pod_execution_role_arns ~} - rolearn: ${role} username: system:node:{{SessionName}} groups: From d0de694b66dd1f69c2ea3d35bef423a1d0d6d5a2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 1 Feb 2022 18:22:25 +0000 Subject: [PATCH 461/962] chore(release): version 18.2.6 [skip ci] ### [18.2.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.5...v18.2.6) (2022-02-01) ### Bug Fixes * Wrong rolearn in aws_auth_configmap_yaml ([#1820](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1820)) ([776009d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/776009d74b16e97974534668ca01a950d660166a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 193d982056..02cfc95adb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.2.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.5...v18.2.6) (2022-02-01) + + +### Bug Fixes + +* Wrong rolearn in aws_auth_configmap_yaml ([#1820](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1820)) ([776009d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/776009d74b16e97974534668ca01a950d660166a)) + ### [18.2.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.4...v18.2.5) (2022-02-01) From a638e4a754c15ab230cfb0e91de026e038ca4e26 Mon Sep 17 00:00:00 2001 From: Devin Young Date: Wed, 2 Feb 2022 10:25:52 -0500 Subject: [PATCH 462/962] fix: Don't tag self managed node security group with kubernetes.io/cluster tag (#1774) --- modules/self-managed-node-group/main.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 8750892e3c..e1e35981fc 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -460,8 +460,7 @@ resource "aws_security_group" "this" { tags = merge( var.tags, { - "Name" = local.security_group_name - "kubernetes.io/cluster/${var.cluster_name}" = "owned" + "Name" = local.security_group_name }, var.security_group_tags ) From d81ff0ddb11ec264a293c44c1d6eabf2aa721f8e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 2 Feb 2022 15:26:35 +0000 Subject: [PATCH 463/962] chore(release): version 18.2.7 [skip ci] ### [18.2.7](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.6...v18.2.7) (2022-02-02) ### Bug Fixes * Don't tag self managed node security group with kubernetes.io/cluster tag ([#1774](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1774)) ([a638e4a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a638e4a754c15ab230cfb0e91de026e038ca4e26)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02cfc95adb..722dd9374c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.2.7](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.6...v18.2.7) (2022-02-02) + + +### Bug Fixes + +* Don't tag self managed node security group with kubernetes.io/cluster tag ([#1774](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1774)) ([a638e4a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a638e4a754c15ab230cfb0e91de026e038ca4e26)) + ### [18.2.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.5...v18.2.6) (2022-02-01) From 9186defcf6ef72502131cffb8b781e1591d2139e Mon Sep 17 00:00:00 2001 From: kahirokunn Date: Thu, 3 Feb 2022 23:45:50 +0900 Subject: [PATCH 464/962] feat: Add launch_template_tags variable for additional launch template tags (#1835) --- modules/eks-managed-node-group/README.md | 1 + modules/eks-managed-node-group/main.tf | 2 +- modules/eks-managed-node-group/variables.tf | 6 ++++++ modules/self-managed-node-group/README.md | 1 + modules/self-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/variables.tf | 6 ++++++ node_groups.tf | 2 ++ 7 files changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 9605e7fef4..b53bf7f05f 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -129,6 +129,7 @@ module "eks_managed_node_group" { | [launch\_template\_default\_version](#input\_launch\_template\_default\_version) | Default version of the launch template | `string` | `null` | no | | [launch\_template\_description](#input\_launch\_template\_description) | Description of the launch template | `string` | `null` | no | | [launch\_template\_name](#input\_launch\_template\_name) | Launch template name - either to be created (`var.create_launch_template` = `true`) or existing (`var.create_launch_template` = `false`) | `string` | `""` | no | +| [launch\_template\_tags](#input\_launch\_template\_tags) | A map of additional tags to add to the tag\_specifications of launch template created | `map(string)` | `{}` | no | | [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | | [launch\_template\_version](#input\_launch\_template\_version) | Launch template version number. The default is `$Default` | `string` | `null` | no | | [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with | `map(string)` | `null` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 71972b22e9..ecb0b7e988 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -238,7 +238,7 @@ resource "aws_launch_template" "this" { for_each = toset(["instance", "volume", "network-interface"]) content { resource_type = tag_specifications.key - tags = merge(var.tags, { Name = var.name }) + tags = merge(var.tags, { Name = var.name }, var.launch_template_tags) } } diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index ddefdcf655..d176ff08d9 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -238,6 +238,12 @@ variable "placement" { default = null } +variable "launch_template_tags" { + description = "A map of additional tags to add to the tag_specifications of launch template created" + type = map(string) + default = {} +} + ################################################################################ # EKS Managed Node Group ################################################################################ diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index b1ff6b64fd..0d705f730e 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -128,6 +128,7 @@ module "self_managed_node_group" { | [launch\_template\_default\_version](#input\_launch\_template\_default\_version) | Default Version of the launch template | `string` | `null` | no | | [launch\_template\_description](#input\_launch\_template\_description) | Description of the launch template | `string` | `null` | no | | [launch\_template\_name](#input\_launch\_template\_name) | Launch template name - either to be created (`var.create_launch_template` = `true`) or existing (`var.create_launch_template` = `false`) | `string` | `null` | no | +| [launch\_template\_tags](#input\_launch\_template\_tags) | A map of additional tags to add to the tag\_specifications of launch template created | `map(string)` | `{}` | no | | [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | | [launch\_template\_version](#input\_launch\_template\_version) | Launch template version. Can be version number, `$Latest`, or `$Default` | `string` | `null` | no | | [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with | `map(string)` | `null` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index e1e35981fc..3f0efc3854 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -229,7 +229,7 @@ resource "aws_launch_template" "this" { for_each = toset(["instance", "volume", "network-interface"]) content { resource_type = tag_specifications.key - tags = merge(var.tags, { Name = var.name }) + tags = merge(var.tags, { Name = var.name }, var.launch_template_tags) } } diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 7bcebeaddc..e9f8646a40 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -250,6 +250,12 @@ variable "metadata_options" { } } +variable "launch_template_tags" { + description = "A map of additional tags to add to the tag_specifications of launch template created" + type = map(string) + default = {} +} + ################################################################################ # Autoscaling group ################################################################################ diff --git a/node_groups.tf b/node_groups.tf index a0884356f0..0381005a24 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -273,6 +273,7 @@ module "eks_managed_node_group" { launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.eks_managed_node_group_defaults.launch_template_use_name_prefix, true) launch_template_version = try(each.value.launch_template_version, var.eks_managed_node_group_defaults.launch_template_version, null) launch_template_description = try(each.value.launch_template_description, var.eks_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} EKS managed node group") + launch_template_tags = try(each.value.launch_template_tags, var.eks_managed_node_group_defaults.launch_template_tags, {}) ebs_optimized = try(each.value.ebs_optimized, var.eks_managed_node_group_defaults.ebs_optimized, null) key_name = try(each.value.key_name, var.eks_managed_node_group_defaults.key_name, null) @@ -390,6 +391,7 @@ module "self_managed_node_group" { launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.self_managed_node_group_defaults.launch_template_use_name_prefix, true) launch_template_version = try(each.value.launch_template_version, var.self_managed_node_group_defaults.launch_template_version, null) launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") + launch_template_tags = try(each.value.launch_template_tags, var.self_managed_node_group_defaults.launch_template_tags, {}) ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") From 2f01a36f4f9e2b30a9a6993270b67bcf86b670b4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 3 Feb 2022 14:46:28 +0000 Subject: [PATCH 465/962] chore(release): version 18.3.0 [skip ci] ## [18.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.7...v18.3.0) (2022-02-03) ### Features * Add launch_template_tags variable for additional launch template tags ([#1835](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1835)) ([9186def](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/9186defcf6ef72502131cffb8b781e1591d2139e)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 722dd9374c..94d7c3866f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.7...v18.3.0) (2022-02-03) + + +### Features + +* Add launch_template_tags variable for additional launch template tags ([#1835](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1835)) ([9186def](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/9186defcf6ef72502131cffb8b781e1591d2139e)) + ### [18.2.7](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.6...v18.2.7) (2022-02-02) From ccc4747122b29ac35975e3c89edaa6ee28a86e4a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 4 Feb 2022 15:08:43 -0500 Subject: [PATCH 466/962] fix: The `block_duration_minutes` attribute under launch template `spot_options` is not a required (#1847) --- examples/complete/main.tf | 4 ++++ modules/eks-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/main.tf | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 67777884d2..85b02b1d15 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -192,6 +192,10 @@ module "eks_managed_node_group" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets + vpc_security_group_ids = [ + module.eks.cluster_primary_security_group_id, + module.eks.cluster_security_group_id, + ] tags = merge(local.tags, { Separate = "eks-managed-node-group" }) } diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index ecb0b7e988..0341806ed5 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -165,7 +165,7 @@ resource "aws_launch_template" "this" { dynamic "spot_options" { for_each = lookup(instance_market_options.value, "spot_options", null) != null ? [instance_market_options.value.spot_options] : [] content { - block_duration_minutes = spot_options.value.block_duration_minutes + block_duration_minutes = lookup(spot_options.value, "block_duration_minutes", null) instance_interruption_behavior = lookup(spot_options.value, "instance_interruption_behavior", null) max_price = lookup(spot_options.value, "max_price", null) spot_instance_type = lookup(spot_options.value, "spot_instance_type", null) diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 3f0efc3854..1b7ca664fd 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -158,7 +158,7 @@ resource "aws_launch_template" "this" { dynamic "spot_options" { for_each = lookup(instance_market_options.value, "spot_options", null) != null ? [instance_market_options.value.spot_options] : [] content { - block_duration_minutes = spot_options.value.block_duration_minutes + block_duration_minutes = lookup(spot_options.value, block_duration_minutes, null) instance_interruption_behavior = lookup(spot_options.value, "instance_interruption_behavior", null) max_price = lookup(spot_options.value, "max_price", null) spot_instance_type = lookup(spot_options.value, "spot_instance_type", null) From 2a47dfec16305573da44800d6390bf431657c3b3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 4 Feb 2022 20:09:19 +0000 Subject: [PATCH 467/962] chore(release): version 18.3.1 [skip ci] ### [18.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.3.0...v18.3.1) (2022-02-04) ### Bug Fixes * The `block_duration_minutes` attribute under launch template `spot_options` is not a required ([#1847](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1847)) ([ccc4747](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ccc4747122b29ac35975e3c89edaa6ee28a86e4a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94d7c3866f..24e30a9e6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.3.0...v18.3.1) (2022-02-04) + + +### Bug Fixes + +* The `block_duration_minutes` attribute under launch template `spot_options` is not a required ([#1847](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1847)) ([ccc4747](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ccc4747122b29ac35975e3c89edaa6ee28a86e4a)) + ## [18.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.2.7...v18.3.0) (2022-02-03) From 21c3802dea52bf51ab99c322fcfdce554086a794 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 6 Feb 2022 14:36:34 -0500 Subject: [PATCH 468/962] feat: enable IRSA by default (#1849) --- README.md | 2 +- examples/complete/main.tf | 2 -- examples/eks_managed_node_group/main.tf | 2 -- examples/fargate_profile/main.tf | 2 -- examples/irsa_autoscale_refresh/main.tf | 2 -- examples/self_managed_node_group/main.tf | 2 -- variables.tf | 2 +- 7 files changed, 2 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 17d4a325db..4223483aa8 100644 --- a/README.md +++ b/README.md @@ -793,7 +793,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | -| [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no | +| [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | | [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no | | [fargate\_profiles](#input\_fargate\_profiles) | Map of Fargate Profile definitions to create | `any` | `{}` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 85b02b1d15..8543f34db1 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -44,8 +44,6 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets - enable_irsa = true - # Extend cluster security group rules cluster_security_group_additional_rules = { egress_nodes_ephemeral_ports_tcp = { diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 2739f2afcb..f985b49c57 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -50,8 +50,6 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets - enable_irsa = true - # Extend cluster security group rules cluster_security_group_additional_rules = { egress_nodes_ephemeral_ports_tcp = { diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index 61fbb43671..6106a272b4 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -45,8 +45,6 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets - enable_irsa = true - # You require a node group to schedule coredns which is critical for running correctly internal DNS. # If you want to use only fargate you must follow docs `(Optional) Update CoreDNS` # available under https://docs.aws.amazon.com/eks/latest/userguide/fargate-getting-started.html diff --git a/examples/irsa_autoscale_refresh/main.tf b/examples/irsa_autoscale_refresh/main.tf index 9e74e3d9fe..6ad656de3f 100644 --- a/examples/irsa_autoscale_refresh/main.tf +++ b/examples/irsa_autoscale_refresh/main.tf @@ -35,8 +35,6 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets - enable_irsa = true - # Self Managed Node Group(s) self_managed_node_groups = { refresh = { diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index b8598d7065..43da796503 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -46,8 +46,6 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets - enable_irsa = true - # Extend cluster security group rules cluster_security_group_additional_rules = { egress_nodes_ephemeral_ports_tcp = { diff --git a/variables.tf b/variables.tf index 5452649421..0d166fd24a 100644 --- a/variables.tf +++ b/variables.tf @@ -238,7 +238,7 @@ variable "node_security_group_tags" { variable "enable_irsa" { description = "Determines whether to create an OpenID Connect Provider for EKS to enable IRSA" type = bool - default = false + default = true } variable "openid_connect_audiences" { From ef22c7988c3b28db341771f700813fb29553a5c1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 6 Feb 2022 19:37:10 +0000 Subject: [PATCH 469/962] chore(release): version 18.4.0 [skip ci] ## [18.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.3.1...v18.4.0) (2022-02-06) ### Features * enable IRSA by default ([#1849](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1849)) ([21c3802](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/21c3802dea52bf51ab99c322fcfdce554086a794)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24e30a9e6c..bd0b60cc26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.3.1...v18.4.0) (2022-02-06) + + +### Features + +* enable IRSA by default ([#1849](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1849)) ([21c3802](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/21c3802dea52bf51ab99c322fcfdce554086a794)) + ### [18.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.3.0...v18.3.1) (2022-02-04) From 2515e0e561509d026fd0d4725ab0bd864e7340f9 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 7 Feb 2022 15:33:35 -0500 Subject: [PATCH 470/962] fix: Add node group dependency for EKS addons resource creation (#1840) --- main.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/main.tf b/main.tf index e7f1f113b8..7fab72e6f9 100644 --- a/main.tf +++ b/main.tf @@ -234,6 +234,16 @@ resource "aws_eks_addon" "this" { ] } + # Note: if an addon needs to be provisioned ahead of a node group users will + # need to create the addon outside of this module until a 2nd addon resource is added + # to the module (here) that is not dependent on node groups + # Or if addon management improves, this dependency can be removed https://github.com/aws/containers-roadmap/issues/1389 + depends_on = [ + module.fargate_profile, + module.eks_managed_node_group, + module.self_managed_node_group, + ] + tags = var.tags } From 6c6841cdb8fdd03feb09ca9e73217ed01a0df4f2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 7 Feb 2022 20:34:21 +0000 Subject: [PATCH 471/962] chore(release): version 18.4.1 [skip ci] ### [18.4.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.4.0...v18.4.1) (2022-02-07) ### Bug Fixes * Add node group dependency for EKS addons resource creation ([#1840](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1840)) ([2515e0e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/2515e0e561509d026fd0d4725ab0bd864e7340f9)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd0b60cc26..0dd9a80716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.4.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.4.0...v18.4.1) (2022-02-07) + + +### Bug Fixes + +* Add node group dependency for EKS addons resource creation ([#1840](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1840)) ([2515e0e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/2515e0e561509d026fd0d4725ab0bd864e7340f9)) + ## [18.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.3.1...v18.4.0) (2022-02-06) From 665f468c1f4839836b1cb5fa5f18ebba17696288 Mon Sep 17 00:00:00 2001 From: PhilippMT <60776105+PhilippMT@users.noreply.github.com> Date: Tue, 8 Feb 2022 10:16:08 +0100 Subject: [PATCH 472/962] feat: Allow conditional creation of node groups to be set within node group definitions (#1848) --- node_groups.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/node_groups.tf b/node_groups.tf index 0381005a24..68ef4e87ea 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -194,6 +194,8 @@ module "fargate_profile" { for_each = { for k, v in var.fargate_profiles : k => v if var.create } + create = try(each.value.create, true) + # Fargate Profile cluster_name = aws_eks_cluster.this[0].name cluster_ip_family = var.cluster_ip_family @@ -226,6 +228,8 @@ module "eks_managed_node_group" { for_each = { for k, v in var.eks_managed_node_groups : k => v if var.create } + create = try(each.value.create, true) + cluster_name = aws_eks_cluster.this[0].name cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, var.cluster_version) cluster_security_group_id = local.cluster_security_group_id @@ -331,6 +335,8 @@ module "self_managed_node_group" { for_each = { for k, v in var.self_managed_node_groups : k => v if var.create } + create = try(each.value.create, true) + cluster_name = aws_eks_cluster.this[0].name cluster_ip_family = var.cluster_ip_family From 43822aff06ffa7ab38f281422a3c5663e4e0b061 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 8 Feb 2022 09:16:54 +0000 Subject: [PATCH 473/962] chore(release): version 18.5.0 [skip ci] ## [18.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.4.1...v18.5.0) (2022-02-08) ### Features * Allow conditional creation of node groups to be set within node group definitions ([#1848](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1848)) ([665f468](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/665f468c1f4839836b1cb5fa5f18ebba17696288)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dd9a80716..6e34665de8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.4.1...v18.5.0) (2022-02-08) + + +### Features + +* Allow conditional creation of node groups to be set within node group definitions ([#1848](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1848)) ([665f468](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/665f468c1f4839836b1cb5fa5f18ebba17696288)) + ### [18.4.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.4.0...v18.4.1) (2022-02-07) From 07841806a49dcae5a8a340aa4133e17d3534058f Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 8 Feb 2022 14:17:25 -0500 Subject: [PATCH 474/962] chore: remove incorrect resource ARNs for cloudwatch event rule (#1854) --- examples/irsa_autoscale_refresh/charts.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/irsa_autoscale_refresh/charts.tf b/examples/irsa_autoscale_refresh/charts.tf index 1ef1195b9e..9d3c1d3d89 100644 --- a/examples/irsa_autoscale_refresh/charts.tf +++ b/examples/irsa_autoscale_refresh/charts.tf @@ -270,7 +270,6 @@ resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_spot" { event_pattern = jsonencode({ "source" : ["aws.ec2"], "detail-type" : ["EC2 Spot Instance Interruption Warning"] - "resources" : [for group in module.eks.self_managed_node_groups : group.autoscaling_group_arn] }) } From c821ba78ca924273d17e9c3b15eae05dd7fb9c94 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 9 Feb 2022 07:31:47 -0500 Subject: [PATCH 475/962] fix: Use existing node security group when one is provided (#1861) --- node_groups.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node_groups.tf b/node_groups.tf index 68ef4e87ea..48cb510176 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -281,7 +281,7 @@ module "eks_managed_node_group" { ebs_optimized = try(each.value.ebs_optimized, var.eks_managed_node_group_defaults.ebs_optimized, null) key_name = try(each.value.key_name, var.eks_managed_node_group_defaults.key_name, null) - vpc_security_group_ids = compact(concat([try(aws_security_group.node[0].id, "")], try(each.value.vpc_security_group_ids, var.eks_managed_node_group_defaults.vpc_security_group_ids, []))) + vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.eks_managed_node_group_defaults.vpc_security_group_ids, []))) launch_template_default_version = try(each.value.launch_template_default_version, var.eks_managed_node_group_defaults.launch_template_default_version, null) update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.eks_managed_node_group_defaults.update_launch_template_default_version, true) disable_api_termination = try(each.value.disable_api_termination, var.eks_managed_node_group_defaults.disable_api_termination, null) @@ -405,7 +405,7 @@ module "self_managed_node_group" { instance_type = try(each.value.instance_type, var.self_managed_node_group_defaults.instance_type, "m6i.large") key_name = try(each.value.key_name, var.self_managed_node_group_defaults.key_name, null) - vpc_security_group_ids = compact(concat([try(aws_security_group.node[0].id, "")], try(each.value.vpc_security_group_ids, var.self_managed_node_group_defaults.vpc_security_group_ids, []))) + vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.self_managed_node_group_defaults.vpc_security_group_ids, []))) cluster_security_group_id = local.cluster_security_group_id launch_template_default_version = try(each.value.launch_template_default_version, var.self_managed_node_group_defaults.launch_template_default_version, null) update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.self_managed_node_group_defaults.update_launch_template_default_version, true) From d0e831b535030cd5896b0a998a11e770a8247877 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 9 Feb 2022 12:32:25 +0000 Subject: [PATCH 476/962] chore(release): version 18.5.1 [skip ci] ### [18.5.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.5.0...v18.5.1) (2022-02-09) ### Bug Fixes * Use existing node security group when one is provided ([#1861](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1861)) ([c821ba7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c821ba78ca924273d17e9c3b15eae05dd7fb9c94)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e34665de8..04833d4eff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.5.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.5.0...v18.5.1) (2022-02-09) + + +### Bug Fixes + +* Use existing node security group when one is provided ([#1861](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1861)) ([c821ba7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c821ba78ca924273d17e9c3b15eae05dd7fb9c94)) + ## [18.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.4.1...v18.5.0) (2022-02-08) From 6a30b73fe44a97ec3d199342d01d6dad366fe7c2 Mon Sep 17 00:00:00 2001 From: Janosch Maier Date: Wed, 9 Feb 2022 13:54:38 +0100 Subject: [PATCH 477/962] docs: Update security group additional rules description for sourcing created security groups (#1860) Co-authored-by: Anton Babenko --- README.md | 4 ++-- variables.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4223483aa8..0f0c566de9 100644 --- a/README.md +++ b/README.md @@ -775,7 +775,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | -| [cluster\_security\_group\_additional\_rules](#input\_cluster\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created | `any` | `{}` | no | +| [cluster\_security\_group\_additional\_rules](#input\_cluster\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created. Set `source_node_security_group = true` inside rules to set the `node_security_group` as source | `any` | `{}` | no | | [cluster\_security\_group\_description](#input\_cluster\_security\_group\_description) | Description of the cluster security group created | `string` | `"EKS cluster security group"` | no | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Existing security group ID to be attached to the cluster. Required if `create_cluster_security_group` = `false` | `string` | `""` | no | | [cluster\_security\_group\_name](#input\_cluster\_security\_group\_name) | Name to use on cluster security group created | `string` | `null` | no | @@ -804,7 +804,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | -| [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created | `any` | `{}` | no | +| [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no | | [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | | [node\_security\_group\_id](#input\_node\_security\_group\_id) | ID of an existing security group to attach to the node groups created | `string` | `""` | no | | [node\_security\_group\_name](#input\_node\_security\_group\_name) | Name to use on node security group created | `string` | `null` | no | diff --git a/variables.tf b/variables.tf index 0d166fd24a..a25866e3f5 100644 --- a/variables.tf +++ b/variables.tf @@ -164,7 +164,7 @@ variable "cluster_security_group_description" { } variable "cluster_security_group_additional_rules" { - description = "List of additional security group rules to add to the cluster security group created" + description = "List of additional security group rules to add to the cluster security group created. Set `source_node_security_group = true` inside rules to set the `node_security_group` as source" type = any default = {} } @@ -220,7 +220,7 @@ variable "node_security_group_description" { } variable "node_security_group_additional_rules" { - description = "List of additional security group rules to add to the node security group created" + description = "List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source" type = any default = {} } From d3b68479dea49076a36e0c39e8c41407f270dcad Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 11 Feb 2022 16:26:36 -0500 Subject: [PATCH 478/962] feat: Add additional output for OIDC provider (issuer URL without leading `https://`) (#1870) --- .pre-commit-config.yaml | 2 +- README.md | 1 + examples/complete/README.md | 1 + examples/complete/outputs.tf | 5 +++++ examples/eks_managed_node_group/README.md | 1 + examples/eks_managed_node_group/outputs.tf | 5 +++++ examples/fargate_profile/README.md | 1 + examples/fargate_profile/outputs.tf | 5 +++++ examples/irsa_autoscale_refresh/README.md | 1 + examples/irsa_autoscale_refresh/outputs.tf | 5 +++++ examples/self_managed_node_group/README.md | 1 + examples/self_managed_node_group/outputs.tf | 5 +++++ outputs.tf | 5 +++++ 13 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 093121e086..8a010fdd41 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.62.3 + rev: v1.64.0 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/README.md b/README.md index 0f0c566de9..8f7aca78c2 100644 --- a/README.md +++ b/README.md @@ -844,6 +844,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | diff --git a/examples/complete/README.md b/examples/complete/README.md index 6fa1f3e8f3..e8831ada9f 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -94,6 +94,7 @@ No inputs. | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index 8a9153c099..0059b252da 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -55,6 +55,11 @@ output "cluster_security_group_arn" { # IRSA ################################################################################ +output "oidc_provider" { + description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" + value = module.eks.oidc_provider +} + output "oidc_provider_arn" { description = "The ARN of the OIDC Provider if `enable_irsa = true`" value = module.eks.oidc_provider_arn diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index dad516ec15..61e793615b 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -127,6 +127,7 @@ No inputs. | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | diff --git a/examples/eks_managed_node_group/outputs.tf b/examples/eks_managed_node_group/outputs.tf index 3e9620157b..c6d9be3bcf 100644 --- a/examples/eks_managed_node_group/outputs.tf +++ b/examples/eks_managed_node_group/outputs.tf @@ -74,6 +74,11 @@ output "node_security_group_id" { # IRSA ################################################################################ +output "oidc_provider" { + description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" + value = module.eks.oidc_provider +} + output "oidc_provider_arn" { description = "The ARN of the OIDC Provider if `enable_irsa = true`" value = module.eks.oidc_provider_arn diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 85bce85ce7..3d58e3a569 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -71,6 +71,7 @@ No inputs. | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | diff --git a/examples/fargate_profile/outputs.tf b/examples/fargate_profile/outputs.tf index 3e9620157b..c6d9be3bcf 100644 --- a/examples/fargate_profile/outputs.tf +++ b/examples/fargate_profile/outputs.tf @@ -74,6 +74,11 @@ output "node_security_group_id" { # IRSA ################################################################################ +output "oidc_provider" { + description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" + value = module.eks.oidc_provider +} + output "oidc_provider_arn" { description = "The ARN of the OIDC Provider if `enable_irsa = true`" value = module.eks.oidc_provider_arn diff --git a/examples/irsa_autoscale_refresh/README.md b/examples/irsa_autoscale_refresh/README.md index d48a2a8edf..ac65d52e29 100644 --- a/examples/irsa_autoscale_refresh/README.md +++ b/examples/irsa_autoscale_refresh/README.md @@ -97,6 +97,7 @@ No inputs. | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | diff --git a/examples/irsa_autoscale_refresh/outputs.tf b/examples/irsa_autoscale_refresh/outputs.tf index 3e9620157b..c6d9be3bcf 100644 --- a/examples/irsa_autoscale_refresh/outputs.tf +++ b/examples/irsa_autoscale_refresh/outputs.tf @@ -74,6 +74,11 @@ output "node_security_group_id" { # IRSA ################################################################################ +output "oidc_provider" { + description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" + value = module.eks.oidc_provider +} + output "oidc_provider_arn" { description = "The ARN of the OIDC Provider if `enable_irsa = true`" value = module.eks.oidc_provider_arn diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 8356943906..580854b78f 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -91,6 +91,7 @@ No inputs. | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | diff --git a/examples/self_managed_node_group/outputs.tf b/examples/self_managed_node_group/outputs.tf index 3e9620157b..c6d9be3bcf 100644 --- a/examples/self_managed_node_group/outputs.tf +++ b/examples/self_managed_node_group/outputs.tf @@ -74,6 +74,11 @@ output "node_security_group_id" { # IRSA ################################################################################ +output "oidc_provider" { + description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" + value = module.eks.oidc_provider +} + output "oidc_provider_arn" { description = "The ARN of the OIDC Provider if `enable_irsa = true`" value = module.eks.oidc_provider_arn diff --git a/outputs.tf b/outputs.tf index 36799b2dc5..e6867b5ce0 100644 --- a/outputs.tf +++ b/outputs.tf @@ -74,6 +74,11 @@ output "node_security_group_id" { # IRSA ################################################################################ +output "oidc_provider" { + description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" + value = try(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://", ""), "") +} + output "oidc_provider_arn" { description = "The ARN of the OIDC Provider if `enable_irsa = true`" value = try(aws_iam_openid_connect_provider.oidc_provider[0].arn, "") From 9c9ac81e3a3a91d52b5341d005de80a05a2fa0e8 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 11 Feb 2022 21:27:10 +0000 Subject: [PATCH 479/962] chore(release): version 18.6.0 [skip ci] ## [18.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.5.1...v18.6.0) (2022-02-11) ### Features * Add additional output for OIDC provider (issuer URL without leading `https://`) ([#1870](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1870)) ([d3b6847](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d3b68479dea49076a36e0c39e8c41407f270dcad)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04833d4eff..d84f7dae79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.5.1...v18.6.0) (2022-02-11) + + +### Features + +* Add additional output for OIDC provider (issuer URL without leading `https://`) ([#1870](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1870)) ([d3b6847](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d3b68479dea49076a36e0c39e8c41407f270dcad)) + ### [18.5.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.5.0...v18.5.1) (2022-02-09) From 74ad4b09b7bbee857c833cb92afe07499356831d Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 15 Feb 2022 08:01:52 -0500 Subject: [PATCH 480/962] fix: Update autoscaling group `tags` -> `tag` to support v4 of AWS provider (#1866) --- examples/irsa_autoscale_refresh/main.tf | 19 ++------- modules/self-managed-node-group/README.md | 3 +- modules/self-managed-node-group/main.tf | 45 ++++++++------------ modules/self-managed-node-group/variables.tf | 8 +--- node_groups.tf | 3 +- 5 files changed, 23 insertions(+), 55 deletions(-) diff --git a/examples/irsa_autoscale_refresh/main.tf b/examples/irsa_autoscale_refresh/main.tf index 6ad656de3f..faf16dc208 100644 --- a/examples/irsa_autoscale_refresh/main.tf +++ b/examples/irsa_autoscale_refresh/main.tf @@ -54,11 +54,7 @@ module "eks" { triggers = ["tag"] } - propogate_tags = [{ - key = "aws-node-termination-handler/managed" - value = true - propagate_at_launch = true - }] + tags = { "aws-node-termination-handler/managed" = "true" } } mixed_instance = { @@ -82,11 +78,7 @@ module "eks" { ] } - propogate_tags = [{ - key = "aws-node-termination-handler/managed" - value = true - propagate_at_launch = true - }] + tags = { "aws-node-termination-handler/managed" = "true" } } spot = { @@ -96,12 +88,7 @@ module "eks" { } bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" - - propogate_tags = [{ - key = "aws-node-termination-handler/managed" - value = true - propagate_at_launch = true - }] + tags = { "aws-node-termination-handler/managed" = "true" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 0d705f730e..8f75be6ffe 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -85,7 +85,7 @@ module "self_managed_node_group" { | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | -| [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | +| [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `""` | no | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `null` | no | @@ -146,7 +146,6 @@ module "self_managed_node_group" { | [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | `string` | `"linux"` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | | [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | -| [propagate\_tags](#input\_propagate\_tags) | A list of tag blocks. Each element should have keys named `key`, `value`, and `propagate_at_launch` | `list(map(string))` | `[]` | no | | [protect\_from\_scale\_in](#input\_protect\_from\_scale\_in) | Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events. | `bool` | `false` | no | | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | | [schedules](#input\_schedules) | Map of autoscaling group schedule to create | `map(any)` | `{}` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 1b7ca664fd..190460139b 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -378,6 +378,23 @@ resource "aws_autoscaling_group" "this" { } } + dynamic "tag" { + for_each = merge( + { + "Name" = var.name + "kubernetes.io/cluster/${var.cluster_name}" = "owned" + "k8s.io/cluster/${var.cluster_name}" = "owned" + }, + var.tags, + ) + + content { + key = tag.key + value = tag.value + propagate_at_launch = true + } + } + timeouts { delete = var.delete_timeout } @@ -388,34 +405,6 @@ resource "aws_autoscaling_group" "this" { desired_capacity ] } - - tags = concat( - [ - { - key = "Name" - value = var.name - propagate_at_launch = true - }, - { - key = "kubernetes.io/cluster/${var.cluster_name}" - value = "owned" - propagate_at_launch = true - }, - { - key = "k8s.io/cluster/${var.cluster_name}" - value = "owned" - propagate_at_launch = true - }, - ], - var.propagate_tags, - [for k, v in var.tags : - { - key = k - value = v - propagate_at_launch = true - } - ] - ) } ################################################################################ diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index e9f8646a40..0eefab722a 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -23,7 +23,7 @@ variable "platform" { variable "cluster_name" { description = "Name of associated EKS cluster" type = string - default = null + default = "" } variable "cluster_endpoint" { @@ -446,12 +446,6 @@ variable "delete_timeout" { default = null } -variable "propagate_tags" { - description = "A list of tag blocks. Each element should have keys named `key`, `value`, and `propagate_at_launch`" - type = list(map(string)) - default = [] -} - ################################################################################ # Autoscaling group schedule ################################################################################ diff --git a/node_groups.tf b/node_groups.tf index 48cb510176..e59159dbbe 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -450,6 +450,5 @@ module "self_managed_node_group" { security_group_rules = try(each.value.security_group_rules, var.self_managed_node_group_defaults.security_group_rules, {}) security_group_tags = try(each.value.security_group_tags, var.self_managed_node_group_defaults.security_group_tags, {}) - tags = merge(var.tags, try(each.value.tags, var.self_managed_node_group_defaults.tags, {})) - propagate_tags = try(each.value.propagate_tags, var.self_managed_node_group_defaults.propagate_tags, []) + tags = merge(var.tags, try(each.value.tags, var.self_managed_node_group_defaults.tags, {})) } From c0571ab612880f07da30d5fbd5296710bd33f4f9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 15 Feb 2022 13:02:33 +0000 Subject: [PATCH 481/962] chore(release): version 18.6.1 [skip ci] ### [18.6.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.6.0...v18.6.1) (2022-02-15) ### Bug Fixes * Update autoscaling group `tags` -> `tag` to support v4 of AWS provider ([#1866](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1866)) ([74ad4b0](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/74ad4b09b7bbee857c833cb92afe07499356831d)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d84f7dae79..ffe2c4e762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.6.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.6.0...v18.6.1) (2022-02-15) + + +### Bug Fixes + +* Update autoscaling group `tags` -> `tag` to support v4 of AWS provider ([#1866](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1866)) ([74ad4b0](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/74ad4b09b7bbee857c833cb92afe07499356831d)) + ## [18.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.5.1...v18.6.0) (2022-02-11) From 3fc9f2d69c32a2536aaee45adbe0c3449d7fc986 Mon Sep 17 00:00:00 2001 From: Fernando Viana <96263878+c6-fviana@users.noreply.github.com> Date: Tue, 15 Feb 2022 10:42:44 -0300 Subject: [PATCH 482/962] feat: Add variable to provide additional OIDC thumbprints (#1865) Co-authored-by: Anton Babenko --- README.md | 1 + main.tf | 2 +- variables.tf | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8f7aca78c2..a1967e46c8 100644 --- a/README.md +++ b/README.md @@ -791,6 +791,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy) | `bool` | `false` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether a an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | +| [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | diff --git a/main.tf b/main.tf index 7fab72e6f9..f124c4e755 100644 --- a/main.tf +++ b/main.tf @@ -156,7 +156,7 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { count = var.create && var.enable_irsa ? 1 : 0 client_id_list = distinct(compact(concat(["sts.${data.aws_partition.current.dns_suffix}"], var.openid_connect_audiences))) - thumbprint_list = [data.tls_certificate.this[0].certificates[0].sha1_fingerprint] + thumbprint_list = concat([data.tls_certificate.this[0].certificates[0].sha1_fingerprint], var.custom_oidc_thumbprints) url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer tags = merge( diff --git a/variables.tf b/variables.tf index a25866e3f5..709bc270bd 100644 --- a/variables.tf +++ b/variables.tf @@ -247,6 +247,12 @@ variable "openid_connect_audiences" { default = [] } +variable "custom_oidc_thumbprints" { + description = "Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s)" + type = list(string) + default = [] +} + ################################################################################ # Cluster IAM Role ################################################################################ From e1a71e8ebe3be450953edf972b78476bcfdb2c10 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 15 Feb 2022 13:43:21 +0000 Subject: [PATCH 483/962] chore(release): version 18.7.0 [skip ci] ## [18.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.6.1...v18.7.0) (2022-02-15) ### Features * Add variable to provide additional OIDC thumbprints ([#1865](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1865)) ([3fc9f2d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3fc9f2d69c32a2536aaee45adbe0c3449d7fc986)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffe2c4e762..9ee0b82040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.6.1...v18.7.0) (2022-02-15) + + +### Features + +* Add variable to provide additional OIDC thumbprints ([#1865](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1865)) ([3fc9f2d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3fc9f2d69c32a2536aaee45adbe0c3449d7fc986)) + ### [18.6.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.6.0...v18.6.1) (2022-02-15) From 8bc6488d559d603b539bc1a9c4eb8c57c529b25e Mon Sep 17 00:00:00 2001 From: Andrew Conti Date: Tue, 15 Feb 2022 12:31:48 -0500 Subject: [PATCH 484/962] fix: Add missing quotes to block_duration_minutes (#1881) --- modules/self-managed-node-group/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 190460139b..40c68c56f5 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -158,7 +158,7 @@ resource "aws_launch_template" "this" { dynamic "spot_options" { for_each = lookup(instance_market_options.value, "spot_options", null) != null ? [instance_market_options.value.spot_options] : [] content { - block_duration_minutes = lookup(spot_options.value, block_duration_minutes, null) + block_duration_minutes = lookup(spot_options.value, "block_duration_minutes", null) instance_interruption_behavior = lookup(spot_options.value, "instance_interruption_behavior", null) max_price = lookup(spot_options.value, "max_price", null) spot_instance_type = lookup(spot_options.value, "spot_instance_type", null) From 8993d85d25530af05c0ccd7fae79d27a2d3217ac Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 15 Feb 2022 17:32:24 +0000 Subject: [PATCH 485/962] chore(release): version 18.7.1 [skip ci] ### [18.7.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.0...v18.7.1) (2022-02-15) ### Bug Fixes * Add missing quotes to block_duration_minutes ([#1881](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1881)) ([8bc6488](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8bc6488d559d603b539bc1a9c4eb8c57c529b25e)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ee0b82040..8cbfbd09d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.7.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.0...v18.7.1) (2022-02-15) + + +### Bug Fixes + +* Add missing quotes to block_duration_minutes ([#1881](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1881)) ([8bc6488](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8bc6488d559d603b539bc1a9c4eb8c57c529b25e)) + ## [18.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.6.1...v18.7.0) (2022-02-15) From 8de02b9ff4690d1bbefb86d3441662b16abb03dd Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 16 Feb 2022 15:23:04 -0500 Subject: [PATCH 486/962] fix: Update examples to show integration and usage of new IRSA submodule (#1882) --- README.md | 83 ++++++++++++++++ examples/eks_managed_node_group/README.md | 1 + examples/eks_managed_node_group/main.tf | 27 ++++- examples/irsa_autoscale_refresh/README.md | 8 +- examples/irsa_autoscale_refresh/charts.tf | 114 +++++----------------- main.tf | 6 +- 6 files changed, 135 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index a1967e46c8..80846bc9f0 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,89 @@ module "eks" { } ``` +### IRSA Integration + +An [IAM role for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) module has been created to work in conjunction with the EKS module. The [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) module has a set of pre-defined IAM policies for common addons/controllers/custom resources to allow users to quickly enable common integrations. Check [`policy.tf`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/modules/iam-role-for-service-accounts-eks/policies.tf) for a list of the policies currently supported. A example of this integration is shown below, and more can be found in the [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/examples/iam-role-for-service-accounts-eks/main.tf example directory): + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + + cluster_name = "example" + cluster_version = "1.21" + + cluster_addons = { + vpc-cni = { + resolve_conflicts = "OVERWRITE" + service_account_role_arn = module.vpc_cni_irsa.iam_role_arn + } + } + + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + + eks_managed_node_group_defaults = { + # We are using the IRSA created below for permissions + # This is a better practice as well so that the nodes do not have the permission, + # only the VPC CNI addon will have the permission + iam_role_attach_cni_policy = false + } + + eks_managed_node_groups = { + default = {} + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} + +module "vpc_cni_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + + role_name = "vpc_cni" + attach_vpc_cni_policy = true + vpc_cni_enable_ipv4 = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-node"] + } + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} + +module "karpenter_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + + role_name = "karpenter_controller" + attach_karpenter_controller_policy = true + + karpenter_controller_cluster_ids = [module.eks.cluster_id] + karpenter_controller_node_iam_role_arns = [ + module.eks.eks_managed_node_groups["default"].iam_role_arn + ] + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["karpenter:karpenter"] + } + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + ## Node Group Configuration ⚠️ The configurations shown below are referenced from within the root EKS module; there will be slight differences in the default values provided when compared to the underlying sub-modules (`eks-managed-node-group`, `self-managed-node-group`, and `fargate-profile`). diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 61e793615b..769a9cbd30 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -75,6 +75,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|--------|---------| | [eks](#module\_eks) | ../.. | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | +| [vpc\_cni\_irsa](#module\_vpc\_cni\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.12 | ## Resources diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index f985b49c57..3ef80700b8 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -29,8 +29,7 @@ module "eks" { cluster_endpoint_public_access = true # IPV6 - cluster_ip_family = "ipv6" - create_cni_ipv6_iam_policy = true + cluster_ip_family = "ipv6" cluster_addons = { coredns = { @@ -38,7 +37,8 @@ module "eks" { } kube-proxy = {} vpc-cni = { - resolve_conflicts = "OVERWRITE" + resolve_conflicts = "OVERWRITE" + service_account_role_arn = module.vpc_cni_irsa.iam_role_arn } } @@ -87,6 +87,9 @@ module "eks" { ami_type = "AL2_x86_64" disk_size = 50 instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] + + # We are using the IRSA created below for permissions + iam_role_attach_cni_policy = false } eks_managed_node_groups = { @@ -421,6 +424,24 @@ module "vpc" { tags = local.tags } +module "vpc_cni_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 4.12" + + role_name_prefix = "VPC-CNI-IRSA" + attach_vpc_cni_policy = true + vpc_cni_enable_ipv6 = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-node"] + } + } + + tags = local.tags +} + resource "aws_security_group" "additional" { name_prefix = "${local.name}-additional" vpc_id = module.vpc.vpc_id diff --git a/examples/irsa_autoscale_refresh/README.md b/examples/irsa_autoscale_refresh/README.md index ac65d52e29..c30a6f6b41 100644 --- a/examples/irsa_autoscale_refresh/README.md +++ b/examples/irsa_autoscale_refresh/README.md @@ -40,10 +40,10 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| -| [aws\_node\_termination\_handler\_role](#module\_aws\_node\_termination\_handler\_role) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | ~> 4.0 | | [aws\_node\_termination\_handler\_sqs](#module\_aws\_node\_termination\_handler\_sqs) | terraform-aws-modules/sqs/aws | ~> 3.0 | +| [cluster\_autoscaler\_irsa](#module\_cluster\_autoscaler\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.12 | | [eks](#module\_eks) | ../.. | n/a | -| [iam\_assumable\_role\_cluster\_autoscaler](#module\_iam\_assumable\_role\_cluster\_autoscaler) | terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc | ~> 4.0 | +| [node\_termination\_handler\_irsa](#module\_node\_termination\_handler\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.12 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources @@ -55,17 +55,13 @@ Note that this example may create resources which cost money. Run `terraform des | [aws_cloudwatch_event_rule.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_target.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | | [aws_cloudwatch_event_target.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | -| [aws_iam_policy.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [helm_release.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [null_resource.apply](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | -| [aws_iam_policy_document.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.aws_node_termination_handler_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | ## Inputs diff --git a/examples/irsa_autoscale_refresh/charts.tf b/examples/irsa_autoscale_refresh/charts.tf index 9d3c1d3d89..956b8de9f4 100644 --- a/examples/irsa_autoscale_refresh/charts.tf +++ b/examples/irsa_autoscale_refresh/charts.tf @@ -32,7 +32,7 @@ resource "helm_release" "cluster_autoscaler" { set { name = "rbac.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.iam_assumable_role_cluster_autoscaler.iam_role_arn + value = module.cluster_autoscaler_irsa.iam_role_arn type = "string" } @@ -57,63 +57,24 @@ resource "helm_release" "cluster_autoscaler" { ] } -module "iam_assumable_role_cluster_autoscaler" { - source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "~> 4.0" +module "cluster_autoscaler_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 4.12" - create_role = true role_name_prefix = "cluster-autoscaler" role_description = "IRSA role for cluster autoscaler" - provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") - role_policy_arns = [aws_iam_policy.cluster_autoscaler.arn] - oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:cluster-autoscaler-aws"] - oidc_fully_qualified_audiences = ["sts.amazonaws.com"] + attach_cluster_autoscaler_policy = true + cluster_autoscaler_cluster_ids = [module.eks.cluster_id] - tags = local.tags -} - -resource "aws_iam_policy" "cluster_autoscaler" { - name = "KarpenterControllerPolicy-refresh" - policy = data.aws_iam_policy_document.cluster_autoscaler.json - - tags = local.tags -} - -data "aws_iam_policy_document" "cluster_autoscaler" { - statement { - sid = "clusterAutoscalerAll" - actions = [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "ec2:DescribeLaunchTemplateVersions", - ] - resources = ["*"] - } - - statement { - sid = "clusterAutoscalerOwn" - actions = [ - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - ] - resources = ["*"] - - condition { - test = "StringEquals" - variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/${module.eks.cluster_id}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled" - values = ["true"] + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:cluster-autoscaler-aws"] } } + + tags = local.tags } ################################################################################ @@ -142,7 +103,7 @@ resource "helm_release" "aws_node_termination_handler" { set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.aws_node_termination_handler_role.iam_role_arn + value = module.node_termination_handler_irsa.iam_role_arn type = "string" } @@ -172,51 +133,24 @@ resource "helm_release" "aws_node_termination_handler" { ] } -module "aws_node_termination_handler_role" { - source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "~> 4.0" +module "node_termination_handler_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 4.12" - create_role = true role_name_prefix = "node-termination-handler" role_description = "IRSA role for node termination handler" - provider_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "") - role_policy_arns = [aws_iam_policy.aws_node_termination_handler.arn] - oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:aws-node-termination-handler"] - oidc_fully_qualified_audiences = ["sts.amazonaws.com"] + attach_node_termination_handler_policy = true + node_termination_handler_sqs_queue_arns = [module.aws_node_termination_handler_sqs.sqs_queue_arn] - tags = local.tags -} - -resource "aws_iam_policy" "aws_node_termination_handler" { - name = "${local.name}-aws-node-termination-handler" - policy = data.aws_iam_policy_document.aws_node_termination_handler.json - - tags = local.tags -} - -data "aws_iam_policy_document" "aws_node_termination_handler" { - statement { - actions = [ - "ec2:DescribeInstances", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeTags", - ] - resources = ["*"] - } - - statement { - actions = ["autoscaling:CompleteLifecycleAction"] - resources = [for group in module.eks.self_managed_node_groups : group.autoscaling_group_arn] + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-node-termination-handler"] + } } - statement { - actions = [ - "sqs:DeleteMessage", - "sqs:ReceiveMessage" - ] - resources = [module.aws_node_termination_handler_sqs.sqs_queue_arn] - } + tags = local.tags } module "aws_node_termination_handler_sqs" { diff --git a/main.tf b/main.tf index f124c4e755..853df4ba1e 100644 --- a/main.tf +++ b/main.tf @@ -183,7 +183,7 @@ data "aws_iam_policy_document" "assume_role_policy" { principals { type = "Service" - identifiers = ["eks.amazonaws.com"] + identifiers = ["eks.${data.aws_partition.current.dns_suffix}"] } } } @@ -234,10 +234,6 @@ resource "aws_eks_addon" "this" { ] } - # Note: if an addon needs to be provisioned ahead of a node group users will - # need to create the addon outside of this module until a 2nd addon resource is added - # to the module (here) that is not dependent on node groups - # Or if addon management improves, this dependency can be removed https://github.com/aws/containers-roadmap/issues/1389 depends_on = [ module.fargate_profile, module.eks_managed_node_group, From 9a99689cc13147f4afc426b34ba009875a28614e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 16 Feb 2022 20:23:37 +0000 Subject: [PATCH 487/962] chore(release): version 18.7.2 [skip ci] ### [18.7.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.1...v18.7.2) (2022-02-16) ### Bug Fixes * Update examples to show integration and usage of new IRSA submodule ([#1882](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1882)) ([8de02b9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8de02b9ff4690d1bbefb86d3441662b16abb03dd)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cbfbd09d4..7a137c467d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.7.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.1...v18.7.2) (2022-02-16) + + +### Bug Fixes + +* Update examples to show integration and usage of new IRSA submodule ([#1882](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1882)) ([8de02b9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8de02b9ff4690d1bbefb86d3441662b16abb03dd)) + ### [18.7.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.0...v18.7.1) (2022-02-15) From 9af0c2495a1fe7a02411ac436f48f6d9ca8b359f Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 2 Mar 2022 12:26:20 -0500 Subject: [PATCH 488/962] fix: Add support for overriding DNS suffix for cluster IAM role service principal endpoint (#1905) --- README.md | 1 + examples/eks_managed_node_group/main.tf | 6 +++++- main.tf | 6 +++++- variables.tf | 8 ++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 80846bc9f0..62ccb3d2a6 100644 --- a/README.md +++ b/README.md @@ -855,6 +855,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `false` | no | | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `true` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |

[
"0.0.0.0/0"
]
| no | +| [cluster\_iam\_role\_dns\_suffix](#input\_cluster\_iam\_role\_dns\_suffix) | Base DNS domain name for the current partition (e.g., amazonaws.com in AWS Commercial, amazonaws.com.cn in AWS China) | `string` | `null` | no | | [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 3ef80700b8..45251fffc8 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -89,7 +89,11 @@ module "eks" { instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] # We are using the IRSA created below for permissions - iam_role_attach_cni_policy = false + # However, we have to deploy with the policy attached FIRST (when creating a fresh cluster) + # and then turn this off after the cluster/node group is created. Without this initial policy, + # the VPC CNI fails to assign IPs and nodes cannot join the cluster + # See https://github.com/aws/containers-roadmap/issues/1666 for more context + iam_role_attach_cni_policy = true } eks_managed_node_groups = { diff --git a/main.tf b/main.tf index 853df4ba1e..918f11f788 100644 --- a/main.tf +++ b/main.tf @@ -172,6 +172,10 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { locals { iam_role_name = coalesce(var.iam_role_name, "${var.cluster_name}-cluster") policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + + # TODO - hopefully this can be removed once the AWS endpoint is named properly in China + # https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1904 + dns_suffix = coalesce(var.cluster_iam_role_dns_suffix, data.aws_partition.current.dns_suffix) } data "aws_iam_policy_document" "assume_role_policy" { @@ -183,7 +187,7 @@ data "aws_iam_policy_document" "assume_role_policy" { principals { type = "Service" - identifiers = ["eks.${data.aws_partition.current.dns_suffix}"] + identifiers = ["eks.${local.dns_suffix}"] } } } diff --git a/variables.tf b/variables.tf index 709bc270bd..8c221a3f55 100644 --- a/variables.tf +++ b/variables.tf @@ -305,6 +305,14 @@ variable "iam_role_additional_policies" { default = [] } +# TODO - hopefully this can be removed once the AWS endpoint is named properly in China +# https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1904 +variable "cluster_iam_role_dns_suffix" { + description = "Base DNS domain name for the current partition (e.g., amazonaws.com in AWS Commercial, amazonaws.com.cn in AWS China)" + type = string + default = null +} + variable "iam_role_tags" { description = "A map of additional tags to add to the IAM role created" type = map(string) From cbd9e4fc0e517f6f671911c0ff8ffa9a6af19eb5 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 2 Mar 2022 17:26:58 +0000 Subject: [PATCH 489/962] chore(release): version 18.7.3 [skip ci] ### [18.7.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.2...v18.7.3) (2022-03-02) ### Bug Fixes * Add support for overriding DNS suffix for cluster IAM role service principal endpoint ([#1905](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1905)) ([9af0c24](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/9af0c2495a1fe7a02411ac436f48f6d9ca8b359f)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a137c467d..6e7c75bcae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.7.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.2...v18.7.3) (2022-03-02) + + +### Bug Fixes + +* Add support for overriding DNS suffix for cluster IAM role service principal endpoint ([#1905](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1905)) ([9af0c24](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/9af0c2495a1fe7a02411ac436f48f6d9ca8b359f)) + ### [18.7.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.1...v18.7.2) (2022-02-16) From 7644952131a466ca22ba5b3e62cd988e01eff716 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 2 Mar 2022 12:29:35 -0500 Subject: [PATCH 490/962] feat: Add additional IAM policy to allow cluster role to use KMS key provided for cluster encryption (#1915) --- README.md | 3 ++ examples/eks_managed_node_group/main.tf | 8 ++++++ main.tf | 38 +++++++++++++++++++++++-- variables.tf | 6 ++++ 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 62ccb3d2a6..1a78ba08bb 100644 --- a/README.md +++ b/README.md @@ -830,8 +830,10 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | | [aws_eks_identity_provider_config.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_identity_provider_config) | resource | | [aws_iam_openid_connect_provider.oidc_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource | +| [aws_iam_policy.cluster_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.cluster_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_security_group.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | @@ -846,6 +848,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [attach\_cluster\_encryption\_policy](#input\_attach\_cluster\_encryption\_policy) | Indicates whether or not to attach an additional policy for the cluster IAM role to utilize the encryption key provided | `bool` | `true` | no | | [cloudwatch\_log\_group\_kms\_key\_id](#input\_cloudwatch\_log\_group\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `null` | no | | [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days | `number` | `90` | no | | [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 45251fffc8..639a56f827 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -31,6 +31,14 @@ module "eks" { # IPV6 cluster_ip_family = "ipv6" + # We are using the IRSA created below for permissions + # However, we have to deploy with the policy attached FIRST (when creating a fresh cluster) + # and then turn this off after the cluster/node group is created. Without this initial policy, + # the VPC CNI fails to assign IPs and nodes cannot join the cluster + # See https://github.com/aws/containers-roadmap/issues/1666 for more context + # TODO - remove this policy once AWS releases a managed version similar to AmazonEKS_CNI_Policy (IPv4) + create_cni_ipv6_iam_policy = true + cluster_addons = { coredns = { resolve_conflicts = "OVERWRITE" diff --git a/main.tf b/main.tf index 918f11f788..450ab6c2ef 100644 --- a/main.tf +++ b/main.tf @@ -30,9 +30,9 @@ resource "aws_eks_cluster" "this" { content { provider { - key_arn = encryption_config.value["provider_key_arn"] + key_arn = encryption_config.value.provider_key_arn } - resources = encryption_config.value["resources"] + resources = encryption_config.value.resources } } @@ -218,6 +218,40 @@ resource "aws_iam_role_policy_attachment" "this" { role = aws_iam_role.this[0].name } +# Using separate attachment due to `The "for_each" value depends on resource attributes that cannot be determined until apply` +resource "aws_iam_role_policy_attachment" "cluster_encryption" { + count = var.create && var.attach_cluster_encryption_policy && length(var.cluster_encryption_config) > 0 ? 1 : 0 + + policy_arn = aws_iam_policy.cluster_encryption[0].arn + role = aws_iam_role.this[0].name +} + +resource "aws_iam_policy" "cluster_encryption" { + count = var.create && var.attach_cluster_encryption_policy && length(var.cluster_encryption_config) > 0 ? 1 : 0 + + name_prefix = "${local.iam_role_name}-ClusterEncryption-" + description = "Cluster encryption policy to allow cluster role to utilize CMK provided" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ListGrants", + "kms:DescribeKey", + ] + Effect = "Allow" + # TODO - does cluster_encryption_config need to be a list?! + Resource = [for config in var.cluster_encryption_config : config.provider_key_arn] + }, + ] + }) + + tags = var.tags +} + ################################################################################ # EKS Addons ################################################################################ diff --git a/variables.tf b/variables.tf index 8c221a3f55..2e479acd22 100644 --- a/variables.tf +++ b/variables.tf @@ -89,6 +89,12 @@ variable "cluster_encryption_config" { default = [] } +variable "attach_cluster_encryption_policy" { + description = "Indicates whether or not to attach an additional policy for the cluster IAM role to utilize the encryption key provided" + type = bool + default = true +} + variable "cluster_tags" { description = "A map of additional tags to add to the cluster" type = map(string) From 54de3823b69bfe10cfe53fadb97f264bad38b772 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 2 Mar 2022 17:30:14 +0000 Subject: [PATCH 491/962] chore(release): version 18.8.0 [skip ci] ## [18.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.3...v18.8.0) (2022-03-02) ### Features * Add additional IAM policy to allow cluster role to use KMS key provided for cluster encryption ([#1915](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1915)) ([7644952](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7644952131a466ca22ba5b3e62cd988e01eff716)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e7c75bcae..a5d7fbbcae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.3...v18.8.0) (2022-03-02) + + +### Features + +* Add additional IAM policy to allow cluster role to use KMS key provided for cluster encryption ([#1915](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1915)) ([7644952](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7644952131a466ca22ba5b3e62cd988e01eff716)) + ### [18.7.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.2...v18.7.3) (2022-03-02) From 0fefca76f2258cee565359e36a4851978602f36d Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 2 Mar 2022 16:10:22 -0500 Subject: [PATCH 492/962] fix: Ensure that cluster encryption policy resources are only relevant when creating the IAM role (#1917) --- main.tf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/main.tf b/main.tf index 450ab6c2ef..7aa426f06b 100644 --- a/main.tf +++ b/main.tf @@ -170,6 +170,7 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { ################################################################################ locals { + create_iam_role = var.create && var.create_iam_role iam_role_name = coalesce(var.iam_role_name, "${var.cluster_name}-cluster") policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" @@ -193,7 +194,7 @@ data "aws_iam_policy_document" "assume_role_policy" { } resource "aws_iam_role" "this" { - count = var.create && var.create_iam_role ? 1 : 0 + count = local.create_iam_role ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}${var.prefix_separator}" : null @@ -209,7 +210,7 @@ resource "aws_iam_role" "this" { # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group resource "aws_iam_role_policy_attachment" "this" { - for_each = var.create && var.create_iam_role ? toset(compact(distinct(concat([ + for_each = local.create_iam_role ? toset(compact(distinct(concat([ "${local.policy_arn_prefix}/AmazonEKSClusterPolicy", "${local.policy_arn_prefix}/AmazonEKSVPCResourceController", ], var.iam_role_additional_policies)))) : toset([]) @@ -220,14 +221,14 @@ resource "aws_iam_role_policy_attachment" "this" { # Using separate attachment due to `The "for_each" value depends on resource attributes that cannot be determined until apply` resource "aws_iam_role_policy_attachment" "cluster_encryption" { - count = var.create && var.attach_cluster_encryption_policy && length(var.cluster_encryption_config) > 0 ? 1 : 0 + count = local.create_iam_role && var.attach_cluster_encryption_policy && length(var.cluster_encryption_config) > 0 ? 1 : 0 policy_arn = aws_iam_policy.cluster_encryption[0].arn role = aws_iam_role.this[0].name } resource "aws_iam_policy" "cluster_encryption" { - count = var.create && var.attach_cluster_encryption_policy && length(var.cluster_encryption_config) > 0 ? 1 : 0 + count = local.create_iam_role && var.attach_cluster_encryption_policy && length(var.cluster_encryption_config) > 0 ? 1 : 0 name_prefix = "${local.iam_role_name}-ClusterEncryption-" description = "Cluster encryption policy to allow cluster role to utilize CMK provided" From 27f99f03a7004407625f61eb030833f98326645d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 2 Mar 2022 21:11:00 +0000 Subject: [PATCH 493/962] chore(release): version 18.8.1 [skip ci] ### [18.8.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.8.0...v18.8.1) (2022-03-02) ### Bug Fixes * Ensure that cluster encryption policy resources are only relevant when creating the IAM role ([#1917](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1917)) ([0fefca7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0fefca76f2258cee565359e36a4851978602f36d)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5d7fbbcae..33d4ed0eed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.8.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.8.0...v18.8.1) (2022-03-02) + + +### Bug Fixes + +* Ensure that cluster encryption policy resources are only relevant when creating the IAM role ([#1917](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1917)) ([0fefca7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0fefca76f2258cee565359e36a4851978602f36d)) + ## [18.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.7.3...v18.8.0) (2022-03-02) From 2df1572b8a031fbd31a845cc5c61f015ec387f56 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 9 Mar 2022 09:13:18 -0500 Subject: [PATCH 494/962] feat: Add variables to allow users to control attributes on `cluster_encryption` IAM policy (#1928) --- README.md | 5 +++++ main.tf | 13 ++++++++----- variables.tf | 30 ++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1a78ba08bb..e46dcefdd7 100644 --- a/README.md +++ b/README.md @@ -855,6 +855,11 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no | | [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | | [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | +| [cluster\_encryption\_policy\_description](#input\_cluster\_encryption\_policy\_description) | Description of the cluster encryption policy created | `string` | `"Cluster encryption policy to allow cluster role to utilize CMK provided"` | no | +| [cluster\_encryption\_policy\_name](#input\_cluster\_encryption\_policy\_name) | Name to use on cluster encryption policy created | `string` | `null` | no | +| [cluster\_encryption\_policy\_path](#input\_cluster\_encryption\_policy\_path) | Cluster encryption policy path | `string` | `null` | no | +| [cluster\_encryption\_policy\_tags](#input\_cluster\_encryption\_policy\_tags) | A map of additional tags to add to the cluster encryption policy created | `map(string)` | `{}` | no | +| [cluster\_encryption\_policy\_use\_name\_prefix](#input\_cluster\_encryption\_policy\_use\_name\_prefix) | Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix | `string` | `true` | no | | [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `false` | no | | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `true` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | diff --git a/main.tf b/main.tf index 7aa426f06b..e47582bc8d 100644 --- a/main.tf +++ b/main.tf @@ -174,6 +174,8 @@ locals { iam_role_name = coalesce(var.iam_role_name, "${var.cluster_name}-cluster") policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + cluster_encryption_policy_name = coalesce(var.cluster_encryption_policy_name, "${local.iam_role_name}-ClusterEncryption") + # TODO - hopefully this can be removed once the AWS endpoint is named properly in China # https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1904 dns_suffix = coalesce(var.cluster_iam_role_dns_suffix, data.aws_partition.current.dns_suffix) @@ -230,8 +232,10 @@ resource "aws_iam_role_policy_attachment" "cluster_encryption" { resource "aws_iam_policy" "cluster_encryption" { count = local.create_iam_role && var.attach_cluster_encryption_policy && length(var.cluster_encryption_config) > 0 ? 1 : 0 - name_prefix = "${local.iam_role_name}-ClusterEncryption-" - description = "Cluster encryption policy to allow cluster role to utilize CMK provided" + name = var.cluster_encryption_policy_use_name_prefix ? null : local.cluster_encryption_policy_name + name_prefix = var.cluster_encryption_policy_use_name_prefix ? local.cluster_encryption_policy_name : null + description = var.cluster_encryption_policy_description + path = var.cluster_encryption_policy_path policy = jsonencode({ Version = "2012-10-17" @@ -243,14 +247,13 @@ resource "aws_iam_policy" "cluster_encryption" { "kms:ListGrants", "kms:DescribeKey", ] - Effect = "Allow" - # TODO - does cluster_encryption_config need to be a list?! + Effect = "Allow" Resource = [for config in var.cluster_encryption_config : config.provider_key_arn] }, ] }) - tags = var.tags + tags = merge(var.tags, var.cluster_encryption_policy_tags) } ################################################################################ diff --git a/variables.tf b/variables.tf index 2e479acd22..7d2af1c716 100644 --- a/variables.tf +++ b/variables.tf @@ -325,6 +325,36 @@ variable "iam_role_tags" { default = {} } +variable "cluster_encryption_policy_use_name_prefix" { + description = "Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix" + type = string + default = true +} + +variable "cluster_encryption_policy_name" { + description = "Name to use on cluster encryption policy created" + type = string + default = null +} + +variable "cluster_encryption_policy_description" { + description = "Description of the cluster encryption policy created" + type = string + default = "Cluster encryption policy to allow cluster role to utilize CMK provided" +} + +variable "cluster_encryption_policy_path" { + description = "Cluster encryption policy path" + type = string + default = null +} + +variable "cluster_encryption_policy_tags" { + description = "A map of additional tags to add to the cluster encryption policy created" + type = map(string) + default = {} +} + ################################################################################ # EKS Addons ################################################################################ From 6fe818d0a350c6eb8b65cb461fe5e480597f2ffe Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 9 Mar 2022 14:14:05 +0000 Subject: [PATCH 495/962] chore(release): version 18.9.0 [skip ci] ## [18.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.8.1...v18.9.0) (2022-03-09) ### Features * Add variables to allow users to control attributes on `cluster_encryption` IAM policy ([#1928](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1928)) ([2df1572](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/2df1572b8a031fbd31a845cc5c61f015ec387f56)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33d4ed0eed..0f6a97ec32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.8.1...v18.9.0) (2022-03-09) + + +### Features + +* Add variables to allow users to control attributes on `cluster_encryption` IAM policy ([#1928](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1928)) ([2df1572](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/2df1572b8a031fbd31a845cc5c61f015ec387f56)) + ### [18.8.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.8.0...v18.8.1) (2022-03-02) From fad350d5bf36a7e39aa3840926b4c9968e9f594c Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Sat, 12 Mar 2022 11:10:02 +0100 Subject: [PATCH 496/962] feat: Made it clear that we stand with Ukraine --- README.md | 10 ++++++++++ main.tf | 22 +++++++++++++--------- variables.tf | 6 ++++++ 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e46dcefdd7..ae0afdce63 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Terraform module which creates AWS EKS (Kubernetes) resources +[![SWUbanner](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner2-direct.svg)](https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md) + ## Available Features - AWS EKS Cluster @@ -905,6 +907,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [node\_security\_group\_use\_name\_prefix](#input\_node\_security\_group\_use\_name\_prefix) | Determines whether node security group name (`node_security_group_name`) is used as a prefix | `string` | `true` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider | `list(string)` | `[]` | no | | [prefix\_separator](#input\_prefix\_separator) | The separator to use between the prefix and the generated timestamp for resource names | `string` | `"-"` | no | +| [putin\_khuylo](#input\_putin\_khuylo) | Do you agree that Putin doesn't respect Ukrainian sovereignty and territorial integrity? More info: https://en.wikipedia.org/wiki/Putin_khuylo! | `bool` | `true` | no | | [self\_managed\_node\_group\_defaults](#input\_self\_managed\_node\_group\_defaults) | Map of self-managed node group default configurations | `any` | `{}` | no | | [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create | `any` | `{}` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the EKS cluster (ENIs) will be provisioned along with the nodes/node groups. Node groups can be deployed within a different set of subnet IDs from within the node group configuration | `list(string)` | `[]` | no | @@ -945,3 +948,10 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws ## License Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-rds-aurora/tree/master/LICENSE) for full details. + +## Additional terms of use for users from Russia and Belarus + +By using the code provided in this repository you agree with the following: +* Russia has [illegally annexed Crimea in 2014](https://en.wikipedia.org/wiki/Annexation_of_Crimea_by_the_Russian_Federation) and [brought the war in Donbas](https://en.wikipedia.org/wiki/War_in_Donbas) followed by [full-scale invasion of Ukraine in 2022](https://en.wikipedia.org/wiki/2022_Russian_invasion_of_Ukraine). +* Russia has brought sorrow and devastations to millions of Ukrainians, killed hundreds of innocent people, damaged thousands of buildings, and forced several million people to flee. +* [Putin khuylo!](https://en.wikipedia.org/wiki/Putin_khuylo!) diff --git a/main.tf b/main.tf index e47582bc8d..1523290f9b 100644 --- a/main.tf +++ b/main.tf @@ -1,11 +1,15 @@ data "aws_partition" "current" {} +locals { + create = var.create && var.putin_khuylo +} + ################################################################################ # Cluster ################################################################################ resource "aws_eks_cluster" "this" { - count = var.create ? 1 : 0 + count = local.create ? 1 : 0 name = var.cluster_name role_arn = try(aws_iam_role.this[0].arn, var.iam_role_arn) @@ -56,7 +60,7 @@ resource "aws_eks_cluster" "this" { } resource "aws_cloudwatch_log_group" "this" { - count = var.create && var.create_cloudwatch_log_group ? 1 : 0 + count = local.create && var.create_cloudwatch_log_group ? 1 : 0 name = "/aws/eks/${var.cluster_name}/cluster" retention_in_days = var.cloudwatch_log_group_retention_in_days @@ -72,7 +76,7 @@ resource "aws_cloudwatch_log_group" "this" { locals { cluster_sg_name = coalesce(var.cluster_security_group_name, "${var.cluster_name}-cluster") - create_cluster_sg = var.create && var.create_cluster_security_group + create_cluster_sg = local.create && var.create_cluster_security_group cluster_security_group_id = local.create_cluster_sg ? aws_security_group.cluster[0].id : var.cluster_security_group_id @@ -147,13 +151,13 @@ resource "aws_security_group_rule" "cluster" { ################################################################################ data "tls_certificate" "this" { - count = var.create && var.enable_irsa ? 1 : 0 + count = local.create && var.enable_irsa ? 1 : 0 url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer } resource "aws_iam_openid_connect_provider" "oidc_provider" { - count = var.create && var.enable_irsa ? 1 : 0 + count = local.create && var.enable_irsa ? 1 : 0 client_id_list = distinct(compact(concat(["sts.${data.aws_partition.current.dns_suffix}"], var.openid_connect_audiences))) thumbprint_list = concat([data.tls_certificate.this[0].certificates[0].sha1_fingerprint], var.custom_oidc_thumbprints) @@ -170,7 +174,7 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { ################################################################################ locals { - create_iam_role = var.create && var.create_iam_role + create_iam_role = local.create && var.create_iam_role iam_role_name = coalesce(var.iam_role_name, "${var.cluster_name}-cluster") policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" @@ -182,7 +186,7 @@ locals { } data "aws_iam_policy_document" "assume_role_policy" { - count = var.create && var.create_iam_role ? 1 : 0 + count = local.create && var.create_iam_role ? 1 : 0 statement { sid = "EKSClusterAssumeRole" @@ -261,7 +265,7 @@ resource "aws_iam_policy" "cluster_encryption" { ################################################################################ resource "aws_eks_addon" "this" { - for_each = { for k, v in var.cluster_addons : k => v if var.create } + for_each = { for k, v in var.cluster_addons : k => v if local.create } cluster_name = aws_eks_cluster.this[0].name addon_name = try(each.value.name, each.key) @@ -291,7 +295,7 @@ resource "aws_eks_addon" "this" { ################################################################################ resource "aws_eks_identity_provider_config" "this" { - for_each = { for k, v in var.cluster_identity_providers : k => v if var.create } + for_each = { for k, v in var.cluster_identity_providers : k => v if local.create } cluster_name = aws_eks_cluster.this[0].name diff --git a/variables.tf b/variables.tf index 7d2af1c716..ed972387a5 100644 --- a/variables.tf +++ b/variables.tf @@ -422,3 +422,9 @@ variable "eks_managed_node_group_defaults" { type = any default = {} } + +variable "putin_khuylo" { + description = "Do you agree that Putin doesn't respect Ukrainian sovereignty and territorial integrity? More info: https://en.wikipedia.org/wiki/Putin_khuylo!" + type = bool + default = true +} From 48906c342bdb7a8472d1f1c177e13a1d85e00fa7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 12 Mar 2022 10:10:46 +0000 Subject: [PATCH 497/962] chore(release): version 18.10.0 [skip ci] ## [18.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.9.0...v18.10.0) (2022-03-12) ### Features * Made it clear that we stand with Ukraine ([fad350d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/fad350d5bf36a7e39aa3840926b4c9968e9f594c)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f6a97ec32..96d566be93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.9.0...v18.10.0) (2022-03-12) + + +### Features + +* Made it clear that we stand with Ukraine ([fad350d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/fad350d5bf36a7e39aa3840926b4c9968e9f594c)) + ## [18.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.8.1...v18.9.0) (2022-03-09) From 5935670503bba3405b53e49ddd88a6451f534d4a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 15 Mar 2022 17:06:15 -0400 Subject: [PATCH 498/962] fix: Compact result of cluster security group to avoid disruptive updates when no security groups are supplied (#1934) --- README.md | 6 +++--- main.tf | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ae0afdce63..94e1d01529 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,13 @@ Users may encounter an error such as `Error: Invalid for_each argument - The "fo This error is due to an upstream issue with [Terraform core](https://github.com/hashicorp/terraform/issues/4149). There are two potential options you can take to help mitigate this issue: 1. Create the dependent resources before the cluster => `terraform apply -target ` and then `terraform apply` for the cluster (or other similar means to just ensure the referenced resources exist before creating the cluster) - - Note: this is the route users will have to take for adding additonal security groups to nodes since there isn't a separate "security group attachment" resource -2. For addtional IAM policies, users can attach the policies outside of the cluster definition as demonstrated below + - Note: this is the route users will have to take for adding additional security groups to nodes since there isn't a separate "security group attachment" resource +2. For additional IAM policies, users can attach the policies outside of the cluster definition as demonstrated below ```hcl resource "aws_iam_role_policy_attachment" "additional" { for_each = module.eks.eks_managed_node_groups - # you could also do the following or any comibination: + # you could also do the following or any combination: # for_each = merge( # module.eks.eks_managed_node_groups, # module.eks.self_managed_node_group, diff --git a/main.tf b/main.tf index 1523290f9b..9baaa09fcd 100644 --- a/main.tf +++ b/main.tf @@ -17,7 +17,7 @@ resource "aws_eks_cluster" "this" { enabled_cluster_log_types = var.cluster_enabled_log_types vpc_config { - security_group_ids = distinct(concat(var.cluster_additional_security_group_ids, [local.cluster_security_group_id])) + security_group_ids = compact(distinct(concat(var.cluster_additional_security_group_ids, [local.cluster_security_group_id]))) subnet_ids = var.subnet_ids endpoint_private_access = var.cluster_endpoint_private_access endpoint_public_access = var.cluster_endpoint_public_access From 66200f1c1d6dca432c2c87b2a657bb59aaed9c68 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 15 Mar 2022 21:06:57 +0000 Subject: [PATCH 499/962] chore(release): version 18.10.1 [skip ci] ### [18.10.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.10.0...v18.10.1) (2022-03-15) ### Bug Fixes * Compact result of cluster security group to avoid disruptive updates when no security groups are supplied ([#1934](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1934)) ([5935670](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5935670503bba3405b53e49ddd88a6451f534d4a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96d566be93..e284898e3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.10.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.10.0...v18.10.1) (2022-03-15) + + +### Bug Fixes + +* Compact result of cluster security group to avoid disruptive updates when no security groups are supplied ([#1934](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1934)) ([5935670](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5935670503bba3405b53e49ddd88a6451f534d4a)) + ## [18.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.9.0...v18.10.0) (2022-03-12) From 06a3469d203fc4344d5f94564762432b5cfd2043 Mon Sep 17 00:00:00 2001 From: Dewen Kong Date: Thu, 17 Mar 2022 08:09:09 -0400 Subject: [PATCH 500/962] fix: Sub-modules output the correct eks worker iam arn when workers utilize custom iam role (#1912) --- modules/eks-managed-node-group/outputs.tf | 2 +- modules/fargate-profile/outputs.tf | 2 +- modules/self-managed-node-group/outputs.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index 476abc8420..334b060ad4 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -66,7 +66,7 @@ output "iam_role_name" { output "iam_role_arn" { description = "The Amazon Resource Name (ARN) specifying the IAM role" - value = try(aws_iam_role.this[0].arn, "") + value = try(aws_iam_role.this[0].arn, var.iam_role_arn) } output "iam_role_unique_id" { diff --git a/modules/fargate-profile/outputs.tf b/modules/fargate-profile/outputs.tf index 0bf2edd180..c8b663e80e 100644 --- a/modules/fargate-profile/outputs.tf +++ b/modules/fargate-profile/outputs.tf @@ -9,7 +9,7 @@ output "iam_role_name" { output "iam_role_arn" { description = "The Amazon Resource Name (ARN) specifying the IAM role" - value = try(aws_iam_role.this[0].arn, "") + value = try(aws_iam_role.this[0].arn, var.iam_role_arn) } output "iam_role_unique_id" { diff --git a/modules/self-managed-node-group/outputs.tf b/modules/self-managed-node-group/outputs.tf index 983c92a80c..eaa0c398c8 100644 --- a/modules/self-managed-node-group/outputs.tf +++ b/modules/self-managed-node-group/outputs.tf @@ -124,7 +124,7 @@ output "iam_role_unique_id" { output "iam_instance_profile_arn" { description = "ARN assigned by AWS to the instance profile" - value = try(aws_iam_instance_profile.this[0].arn, "") + value = try(aws_iam_instance_profile.this[0].arn, var.iam_instance_profile_arn) } output "iam_instance_profile_id" { From 2bd941864feaa014a0363075d2f065c74f8e4602 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 17 Mar 2022 12:09:50 +0000 Subject: [PATCH 501/962] chore(release): version 18.10.2 [skip ci] ### [18.10.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.10.1...v18.10.2) (2022-03-17) ### Bug Fixes * Sub-modules output the correct eks worker iam arn when workers utilize custom iam role ([#1912](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1912)) ([06a3469](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/06a3469d203fc4344d5f94564762432b5cfd2043)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e284898e3e..4dd48ed37f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.10.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.10.1...v18.10.2) (2022-03-17) + + +### Bug Fixes + +* Sub-modules output the correct eks worker iam arn when workers utilize custom iam role ([#1912](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1912)) ([06a3469](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/06a3469d203fc4344d5f94564762432b5cfd2043)) + ### [18.10.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.10.0...v18.10.1) (2022-03-15) From f5511e4df1f06954229f48df1cf87f9ebc8da26a Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Thu, 17 Mar 2022 14:57:15 +0100 Subject: [PATCH 502/962] chore: Updated information for users from Russia/Belarus --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 94e1d01529..17fd4f7070 100644 --- a/README.md +++ b/README.md @@ -949,9 +949,8 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-rds-aurora/tree/master/LICENSE) for full details. -## Additional terms of use for users from Russia and Belarus +## Additional information for users from Russia and Belarus -By using the code provided in this repository you agree with the following: * Russia has [illegally annexed Crimea in 2014](https://en.wikipedia.org/wiki/Annexation_of_Crimea_by_the_Russian_Federation) and [brought the war in Donbas](https://en.wikipedia.org/wiki/War_in_Donbas) followed by [full-scale invasion of Ukraine in 2022](https://en.wikipedia.org/wiki/2022_Russian_invasion_of_Ukraine). * Russia has brought sorrow and devastations to millions of Ukrainians, killed hundreds of innocent people, damaged thousands of buildings, and forced several million people to flee. * [Putin khuylo!](https://en.wikipedia.org/wiki/Putin_khuylo!) From a9d2cc8246128fc7f426f0b4596c6799ecf94d8a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 18 Mar 2022 07:29:17 -0400 Subject: [PATCH 503/962] feat: Allow users to specify default launch template name in node groups (#1946) --- node_groups.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node_groups.tf b/node_groups.tf index e59159dbbe..c72a14da8e 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -273,7 +273,7 @@ module "eks_managed_node_group" { # Launch Template create_launch_template = try(each.value.create_launch_template, var.eks_managed_node_group_defaults.create_launch_template, true) - launch_template_name = try(each.value.launch_template_name, each.key) + launch_template_name = try(each.value.launch_template_name, var.eks_managed_node_group_defaults.launch_template_name, each.key) launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.eks_managed_node_group_defaults.launch_template_use_name_prefix, true) launch_template_version = try(each.value.launch_template_version, var.eks_managed_node_group_defaults.launch_template_version, null) launch_template_description = try(each.value.launch_template_description, var.eks_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} EKS managed node group") @@ -393,7 +393,7 @@ module "self_managed_node_group" { # Launch Template create_launch_template = try(each.value.create_launch_template, var.self_managed_node_group_defaults.create_launch_template, true) - launch_template_name = try(each.value.launch_template_name, each.key) + launch_template_name = try(each.value.launch_template_name, var.self_managed_node_group_defaults.launch_template_name, each.key) launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.self_managed_node_group_defaults.launch_template_use_name_prefix, true) launch_template_version = try(each.value.launch_template_version, var.self_managed_node_group_defaults.launch_template_version, null) launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") From 6aefb829c5fddea809f29f10676673e926c39c55 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 18 Mar 2022 11:30:03 +0000 Subject: [PATCH 504/962] chore(release): version 18.11.0 [skip ci] ## [18.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.10.2...v18.11.0) (2022-03-18) ### Features * Allow users to specify default launch template name in node groups ([#1946](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1946)) ([a9d2cc8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a9d2cc8246128fc7f426f0b4596c6799ecf94d8a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dd48ed37f..61637188a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.10.2...v18.11.0) (2022-03-18) + + +### Features + +* Allow users to specify default launch template name in node groups ([#1946](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1946)) ([a9d2cc8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a9d2cc8246128fc7f426f0b4596c6799ecf94d8a)) + ### [18.10.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.10.1...v18.10.2) (2022-03-17) From dd97344d2b9939c01851031f77ac864de5dd6e03 Mon Sep 17 00:00:00 2001 From: Adam Blasko Date: Tue, 22 Mar 2022 11:25:56 +0100 Subject: [PATCH 505/962] chore: Fixed typo in readme (#1951) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 17fd4f7070..3f63bf92d8 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ module "eks" { ### IRSA Integration -An [IAM role for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) module has been created to work in conjunction with the EKS module. The [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) module has a set of pre-defined IAM policies for common addons/controllers/custom resources to allow users to quickly enable common integrations. Check [`policy.tf`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/modules/iam-role-for-service-accounts-eks/policies.tf) for a list of the policies currently supported. A example of this integration is shown below, and more can be found in the [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/examples/iam-role-for-service-accounts-eks/main.tf example directory): +An [IAM role for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) module has been created to work in conjunction with the EKS module. The [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) module has a set of pre-defined IAM policies for common addons/controllers/custom resources to allow users to quickly enable common integrations. Check [`policy.tf`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/modules/iam-role-for-service-accounts-eks/policies.tf) for a list of the policies currently supported. A example of this integration is shown below, and more can be found in the [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/examples/iam-role-for-service-accounts-eks/main.tf) example directory: ```hcl module "eks" { From 8b03b7b85ef80db5de766827ef65b700317c68e6 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 22 Mar 2022 17:26:00 -0400 Subject: [PATCH 506/962] feat: Add outputs for autoscaling group names created to aid in autoscaling group tagging (#1953) --- README.md | 2 ++ examples/complete/README.md | 2 ++ examples/complete/outputs.tf | 10 ++++++++++ examples/eks_managed_node_group/README.md | 2 ++ examples/eks_managed_node_group/outputs.tf | 10 ++++++++++ examples/fargate_profile/README.md | 2 ++ examples/fargate_profile/outputs.tf | 10 ++++++++++ examples/irsa_autoscale_refresh/README.md | 2 ++ examples/irsa_autoscale_refresh/outputs.tf | 10 ++++++++++ examples/self_managed_node_group/README.md | 2 ++ examples/self_managed_node_group/outputs.tf | 10 ++++++++++ modules/eks-managed-node-group/README.md | 1 + modules/eks-managed-node-group/outputs.tf | 5 +++++ outputs.tf | 10 ++++++++++ 14 files changed, 78 insertions(+) diff --git a/README.md b/README.md index 3f63bf92d8..efd4c6686d 100644 --- a/README.md +++ b/README.md @@ -937,12 +937,14 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | +| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | ## License diff --git a/examples/complete/README.md b/examples/complete/README.md index e8831ada9f..a432ec9c47 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -93,8 +93,10 @@ No inputs. | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | +| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index 0059b252da..cbfae9a452 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -134,6 +134,11 @@ output "eks_managed_node_groups" { value = module.eks.eks_managed_node_groups } +output "eks_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by EKS managed node groups" + value = module.eks.eks_managed_node_groups_autoscaling_group_names +} + ################################################################################ # Self Managed Node Group ################################################################################ @@ -143,6 +148,11 @@ output "self_managed_node_groups" { value = module.eks.self_managed_node_groups } +output "self_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by self-managed node groups" + value = module.eks.self_managed_node_groups_autoscaling_group_names +} + ################################################################################ # Additional ################################################################################ diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 769a9cbd30..33dd2c6ebf 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -125,10 +125,12 @@ No inputs. | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | +| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | diff --git a/examples/eks_managed_node_group/outputs.tf b/examples/eks_managed_node_group/outputs.tf index c6d9be3bcf..bfe53985b1 100644 --- a/examples/eks_managed_node_group/outputs.tf +++ b/examples/eks_managed_node_group/outputs.tf @@ -153,6 +153,11 @@ output "eks_managed_node_groups" { value = module.eks.eks_managed_node_groups } +output "eks_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by EKS managed node groups" + value = module.eks.eks_managed_node_groups_autoscaling_group_names +} + ################################################################################ # Self Managed Node Group ################################################################################ @@ -162,6 +167,11 @@ output "self_managed_node_groups" { value = module.eks.self_managed_node_groups } +output "self_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by self-managed node groups" + value = module.eks.self_managed_node_groups_autoscaling_group_names +} + ################################################################################ # Additional ################################################################################ diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 3d58e3a569..cec6ef345b 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -68,10 +68,12 @@ No inputs. | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | +| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | diff --git a/examples/fargate_profile/outputs.tf b/examples/fargate_profile/outputs.tf index c6d9be3bcf..bfe53985b1 100644 --- a/examples/fargate_profile/outputs.tf +++ b/examples/fargate_profile/outputs.tf @@ -153,6 +153,11 @@ output "eks_managed_node_groups" { value = module.eks.eks_managed_node_groups } +output "eks_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by EKS managed node groups" + value = module.eks.eks_managed_node_groups_autoscaling_group_names +} + ################################################################################ # Self Managed Node Group ################################################################################ @@ -162,6 +167,11 @@ output "self_managed_node_groups" { value = module.eks.self_managed_node_groups } +output "self_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by self-managed node groups" + value = module.eks.self_managed_node_groups_autoscaling_group_names +} + ################################################################################ # Additional ################################################################################ diff --git a/examples/irsa_autoscale_refresh/README.md b/examples/irsa_autoscale_refresh/README.md index c30a6f6b41..a331d6973f 100644 --- a/examples/irsa_autoscale_refresh/README.md +++ b/examples/irsa_autoscale_refresh/README.md @@ -90,10 +90,12 @@ No inputs. | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | +| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | diff --git a/examples/irsa_autoscale_refresh/outputs.tf b/examples/irsa_autoscale_refresh/outputs.tf index c6d9be3bcf..bfe53985b1 100644 --- a/examples/irsa_autoscale_refresh/outputs.tf +++ b/examples/irsa_autoscale_refresh/outputs.tf @@ -153,6 +153,11 @@ output "eks_managed_node_groups" { value = module.eks.eks_managed_node_groups } +output "eks_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by EKS managed node groups" + value = module.eks.eks_managed_node_groups_autoscaling_group_names +} + ################################################################################ # Self Managed Node Group ################################################################################ @@ -162,6 +167,11 @@ output "self_managed_node_groups" { value = module.eks.self_managed_node_groups } +output "self_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by self-managed node groups" + value = module.eks.self_managed_node_groups_autoscaling_group_names +} + ################################################################################ # Additional ################################################################################ diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 580854b78f..f91bbf6b4c 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -88,10 +88,12 @@ No inputs. | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | +| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | diff --git a/examples/self_managed_node_group/outputs.tf b/examples/self_managed_node_group/outputs.tf index c6d9be3bcf..bfe53985b1 100644 --- a/examples/self_managed_node_group/outputs.tf +++ b/examples/self_managed_node_group/outputs.tf @@ -153,6 +153,11 @@ output "eks_managed_node_groups" { value = module.eks.eks_managed_node_groups } +output "eks_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by EKS managed node groups" + value = module.eks.eks_managed_node_groups_autoscaling_group_names +} + ################################################################################ # Self Managed Node Group ################################################################################ @@ -162,6 +167,11 @@ output "self_managed_node_groups" { value = module.eks.self_managed_node_groups } +output "self_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by self-managed node groups" + value = module.eks.self_managed_node_groups_autoscaling_group_names +} + ################################################################################ # Additional ################################################################################ diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index b53bf7f05f..f39bf027e9 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -171,6 +171,7 @@ module "eks_managed_node_group" { | [launch\_template\_id](#output\_launch\_template\_id) | The ID of the launch template | | [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | | [node\_group\_arn](#output\_node\_group\_arn) | Amazon Resource Name (ARN) of the EKS Node Group | +| [node\_group\_autoscaling\_group\_names](#output\_node\_group\_autoscaling\_group\_names) | List of the autoscaling group names | | [node\_group\_id](#output\_node\_group\_id) | EKS Cluster name and EKS Node Group name separated by a colon (`:`) | | [node\_group\_resources](#output\_node\_group\_resources) | List of objects containing information about underlying resources | | [node\_group\_status](#output\_node\_group\_status) | Status of the EKS Node Group | diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index 334b060ad4..a73ea1eacf 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -36,6 +36,11 @@ output "node_group_resources" { value = try(aws_eks_node_group.this[0].resources, "") } +output "node_group_autoscaling_group_names" { + description = "List of the autoscaling group names" + value = try(flatten(aws_eks_node_group.this[0].resources[*].autoscaling_groups[*].name), []) +} + output "node_group_status" { description = "Status of the EKS Node Group" value = try(aws_eks_node_group.this[0].arn, "") diff --git a/outputs.tf b/outputs.tf index e6867b5ce0..64eb97da27 100644 --- a/outputs.tf +++ b/outputs.tf @@ -153,6 +153,11 @@ output "eks_managed_node_groups" { value = module.eks_managed_node_group } +output "eks_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by EKS managed node groups" + value = flatten([for group in module.eks_managed_node_group : group.node_group_autoscaling_group_names]) +} + ################################################################################ # Self Managed Node Group ################################################################################ @@ -162,6 +167,11 @@ output "self_managed_node_groups" { value = module.self_managed_node_group } +output "self_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by self-managed node groups" + value = [for group in module.self_managed_node_group : group.autoscaling_group_name] +} + ################################################################################ # Additional ################################################################################ From 9528619acacff3441f5361ae6367fe9074ffde10 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 22 Mar 2022 21:26:43 +0000 Subject: [PATCH 507/962] chore(release): version 18.12.0 [skip ci] ## [18.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.11.0...v18.12.0) (2022-03-22) ### Features * Add outputs for autoscaling group names created to aid in autoscaling group tagging ([#1953](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1953)) ([8b03b7b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8b03b7b85ef80db5de766827ef65b700317c68e6)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61637188a8..32736a571a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.11.0...v18.12.0) (2022-03-22) + + +### Features + +* Add outputs for autoscaling group names created to aid in autoscaling group tagging ([#1953](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1953)) ([8b03b7b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8b03b7b85ef80db5de766827ef65b700317c68e6)) + ## [18.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.10.2...v18.11.0) (2022-03-18) From e21db83d8ff3cd1d3f49acc611931e8917d0b6f8 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 23 Mar 2022 10:41:39 -0400 Subject: [PATCH 508/962] feat: Allow users to selectively attach the EKS created cluster primary security group to nodes (#1952) --- README.md | 1 + examples/complete/main.tf | 16 +++++---- modules/eks-managed-node-group/README.md | 1 + modules/eks-managed-node-group/main.tf | 2 +- modules/eks-managed-node-group/variables.tf | 6 ++++ modules/self-managed-node-group/README.md | 1 + modules/self-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/variables.tf | 6 ++++ node_groups.tf | 36 +++++++++++--------- 9 files changed, 45 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index efd4c6686d..c29d5c9b4b 100644 --- a/README.md +++ b/README.md @@ -658,6 +658,7 @@ Module provided default templates can be found under the [templates directory](h - Users have the ability to opt out of the security group creation and instead provide their own externally created security group if so desired - The security group that is created is designed to handle the bare minimum communication necessary between the control plane and the nodes, as well as any external egress to allow the cluster to successfully launch without error - Users also have the option to supply additional, externally created security groups to the cluster as well via the `cluster_additional_security_group_ids` variable + - Lastly, users are able to opt in to attaching the primary security group automatically created by the EKS service by setting `attach_cluster_primary_security_group` = `true` from the root module for the respective node group (or set it within the node group defaults). This security group is not managed by the module; it is created by the EKS service. It permits all traffic within the domain of the security group as well as all egress traffic to the internet. - Node Group Security Group(s) - Each node group (EKS Managed Node Group and Self Managed Node Group) by default creates its own security group. By default, this security group does not contain any additional security group rules. It is merely an "empty container" that offers users the ability to opt into any addition inbound our outbound rules as necessary diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 8543f34db1..bc519d2f3c 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -108,10 +108,12 @@ module "eks" { # EKS Managed Node Group(s) eks_managed_node_group_defaults = { - ami_type = "AL2_x86_64" - disk_size = 50 - instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] - vpc_security_group_ids = [aws_security_group.additional.id] + ami_type = "AL2_x86_64" + disk_size = 50 + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] + + attach_cluster_primary_security_group = true + vpc_security_group_ids = [aws_security_group.additional.id] } eks_managed_node_groups = { @@ -188,10 +190,10 @@ module "eks_managed_node_group" { cluster_name = module.eks.cluster_id cluster_version = local.cluster_version - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id vpc_security_group_ids = [ - module.eks.cluster_primary_security_group_id, module.eks.cluster_security_group_id, ] diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index f39bf027e9..a89c5f9df0 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -93,6 +93,7 @@ module "eks_managed_node_group" { | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | +| [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes version. Defaults to EKS Cluster Kubernetes version | `string` | `null` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 0341806ed5..662b2aa258 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -54,7 +54,7 @@ resource "aws_launch_template" "this" { key_name = var.key_name user_data = module.user_data.user_data - vpc_security_group_ids = compact(concat([try(aws_security_group.this[0].id, "")], var.vpc_security_group_ids)) + vpc_security_group_ids = compact(concat([try(aws_security_group.this[0].id, ""), var.cluster_primary_security_group_id], var.vpc_security_group_ids)) default_version = var.launch_template_default_version update_default_version = var.update_launch_template_default_version diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index d176ff08d9..93e9555f67 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -126,6 +126,12 @@ variable "vpc_security_group_ids" { default = [] } +variable "cluster_primary_security_group_id" { + description = "The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service" + type = string + default = null +} + variable "launch_template_default_version" { description = "Default version of the launch template" type = string diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 8f75be6ffe..d42a46ac76 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -86,6 +86,7 @@ module "self_managed_node_group" { | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `""` | no | +| [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `null` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 40c68c56f5..40dc4f9a3c 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -57,7 +57,7 @@ resource "aws_launch_template" "this" { key_name = var.key_name user_data = module.user_data.user_data - vpc_security_group_ids = compact(concat([try(aws_security_group.this[0].id, "")], var.vpc_security_group_ids)) + vpc_security_group_ids = compact(concat([try(aws_security_group.this[0].id, ""), var.cluster_primary_security_group_id], var.vpc_security_group_ids)) default_version = var.launch_template_default_version update_default_version = var.update_launch_template_default_version diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 0eefab722a..7591c77ef6 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -234,6 +234,12 @@ variable "vpc_security_group_ids" { default = [] } +variable "cluster_primary_security_group_id" { + description = "The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service" + type = string + default = null +} + variable "enable_monitoring" { description = "Enables/disables detailed monitoring" type = bool diff --git a/node_groups.tf b/node_groups.tf index c72a14da8e..f80e444415 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -281,7 +281,6 @@ module "eks_managed_node_group" { ebs_optimized = try(each.value.ebs_optimized, var.eks_managed_node_group_defaults.ebs_optimized, null) key_name = try(each.value.key_name, var.eks_managed_node_group_defaults.key_name, null) - vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.eks_managed_node_group_defaults.vpc_security_group_ids, []))) launch_template_default_version = try(each.value.launch_template_default_version, var.eks_managed_node_group_defaults.launch_template_default_version, null) update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.eks_managed_node_group_defaults.update_launch_template_default_version, true) disable_api_termination = try(each.value.disable_api_termination, var.eks_managed_node_group_defaults.disable_api_termination, null) @@ -315,13 +314,15 @@ module "eks_managed_node_group" { iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.eks_managed_node_group_defaults.iam_role_additional_policies, []) # Security group - create_security_group = try(each.value.create_security_group, var.eks_managed_node_group_defaults.create_security_group, true) - security_group_name = try(each.value.security_group_name, var.eks_managed_node_group_defaults.security_group_name, null) - security_group_use_name_prefix = try(each.value.security_group_use_name_prefix, var.eks_managed_node_group_defaults.security_group_use_name_prefix, true) - security_group_description = try(each.value.security_group_description, var.eks_managed_node_group_defaults.security_group_description, "EKS managed node group security group") - vpc_id = try(each.value.vpc_id, var.eks_managed_node_group_defaults.vpc_id, var.vpc_id) - security_group_rules = try(each.value.security_group_rules, var.eks_managed_node_group_defaults.security_group_rules, {}) - security_group_tags = try(each.value.security_group_tags, var.eks_managed_node_group_defaults.security_group_tags, {}) + vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.eks_managed_node_group_defaults.vpc_security_group_ids, []))) + cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, var.eks_managed_node_group_defaults.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null + create_security_group = try(each.value.create_security_group, var.eks_managed_node_group_defaults.create_security_group, true) + security_group_name = try(each.value.security_group_name, var.eks_managed_node_group_defaults.security_group_name, null) + security_group_use_name_prefix = try(each.value.security_group_use_name_prefix, var.eks_managed_node_group_defaults.security_group_use_name_prefix, true) + security_group_description = try(each.value.security_group_description, var.eks_managed_node_group_defaults.security_group_description, "EKS managed node group security group") + vpc_id = try(each.value.vpc_id, var.eks_managed_node_group_defaults.vpc_id, var.vpc_id) + security_group_rules = try(each.value.security_group_rules, var.eks_managed_node_group_defaults.security_group_rules, {}) + security_group_tags = try(each.value.security_group_tags, var.eks_managed_node_group_defaults.security_group_tags, {}) tags = merge(var.tags, try(each.value.tags, var.eks_managed_node_group_defaults.tags, {})) } @@ -405,8 +406,6 @@ module "self_managed_node_group" { instance_type = try(each.value.instance_type, var.self_managed_node_group_defaults.instance_type, "m6i.large") key_name = try(each.value.key_name, var.self_managed_node_group_defaults.key_name, null) - vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.self_managed_node_group_defaults.vpc_security_group_ids, []))) - cluster_security_group_id = local.cluster_security_group_id launch_template_default_version = try(each.value.launch_template_default_version, var.self_managed_node_group_defaults.launch_template_default_version, null) update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.self_managed_node_group_defaults.update_launch_template_default_version, true) disable_api_termination = try(each.value.disable_api_termination, var.self_managed_node_group_defaults.disable_api_termination, null) @@ -442,13 +441,16 @@ module "self_managed_node_group" { iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.self_managed_node_group_defaults.iam_role_additional_policies, []) # Security group - create_security_group = try(each.value.create_security_group, var.self_managed_node_group_defaults.create_security_group, true) - security_group_name = try(each.value.security_group_name, var.self_managed_node_group_defaults.security_group_name, null) - security_group_use_name_prefix = try(each.value.security_group_use_name_prefix, var.self_managed_node_group_defaults.security_group_use_name_prefix, true) - security_group_description = try(each.value.security_group_description, var.self_managed_node_group_defaults.security_group_description, "Self managed node group security group") - vpc_id = try(each.value.vpc_id, var.self_managed_node_group_defaults.vpc_id, var.vpc_id) - security_group_rules = try(each.value.security_group_rules, var.self_managed_node_group_defaults.security_group_rules, {}) - security_group_tags = try(each.value.security_group_tags, var.self_managed_node_group_defaults.security_group_tags, {}) + vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.self_managed_node_group_defaults.vpc_security_group_ids, []))) + cluster_security_group_id = local.cluster_security_group_id + cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, var.self_managed_node_group_defaults.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null + create_security_group = try(each.value.create_security_group, var.self_managed_node_group_defaults.create_security_group, true) + security_group_name = try(each.value.security_group_name, var.self_managed_node_group_defaults.security_group_name, null) + security_group_use_name_prefix = try(each.value.security_group_use_name_prefix, var.self_managed_node_group_defaults.security_group_use_name_prefix, true) + security_group_description = try(each.value.security_group_description, var.self_managed_node_group_defaults.security_group_description, "Self managed node group security group") + vpc_id = try(each.value.vpc_id, var.self_managed_node_group_defaults.vpc_id, var.vpc_id) + security_group_rules = try(each.value.security_group_rules, var.self_managed_node_group_defaults.security_group_rules, {}) + security_group_tags = try(each.value.security_group_tags, var.self_managed_node_group_defaults.security_group_tags, {}) tags = merge(var.tags, try(each.value.tags, var.self_managed_node_group_defaults.tags, {})) } From ca7854a49ce3025842a8a67b8d08560ecdc13bb1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 23 Mar 2022 14:42:47 +0000 Subject: [PATCH 509/962] chore(release): version 18.13.0 [skip ci] ## [18.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.12.0...v18.13.0) (2022-03-23) ### Features * Allow users to selectively attach the EKS created cluster primary security group to nodes ([#1952](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1952)) ([e21db83](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e21db83d8ff3cd1d3f49acc611931e8917d0b6f8)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32736a571a..f4393f1571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.12.0...v18.13.0) (2022-03-23) + + +### Features + +* Allow users to selectively attach the EKS created cluster primary security group to nodes ([#1952](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1952)) ([e21db83](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e21db83d8ff3cd1d3f49acc611931e8917d0b6f8)) + ## [18.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.11.0...v18.12.0) (2022-03-22) From 9371a2943b13cc2d9ceb34aef14ec2ccee1cb721 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 24 Mar 2022 03:05:59 -0400 Subject: [PATCH 510/962] feat: Add tags to EKS created cluster security group to match rest of module tagging scheme (#1957) --- README.md | 1 + main.tf | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/README.md b/README.md index c29d5c9b4b..d4c7863f4d 100644 --- a/README.md +++ b/README.md @@ -829,6 +829,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | Name | Type | |------|------| | [aws_cloudwatch_log_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_ec2_tag.cluster_primary_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ec2_tag) | resource | | [aws_eks_addon.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | | [aws_eks_identity_provider_config.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_identity_provider_config) | resource | diff --git a/main.tf b/main.tf index 9baaa09fcd..ad3504ba69 100644 --- a/main.tf +++ b/main.tf @@ -59,6 +59,14 @@ resource "aws_eks_cluster" "this" { ] } +resource "aws_ec2_tag" "cluster_primary_security_group" { + for_each = { for k, v in merge(var.tags, var.cluster_tags) : k => v if var.create } + + resource_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id + key = each.key + value = each.value +} + resource "aws_cloudwatch_log_group" "this" { count = local.create && var.create_cloudwatch_log_group ? 1 : 0 From 7369f8a19e3966b1edafa9cead972fbad6b56f80 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 24 Mar 2022 07:06:35 +0000 Subject: [PATCH 511/962] chore(release): version 18.14.0 [skip ci] ## [18.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.13.0...v18.14.0) (2022-03-24) ### Features * Add tags to EKS created cluster security group to match rest of module tagging scheme ([#1957](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1957)) ([9371a29](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/9371a2943b13cc2d9ceb34aef14ec2ccee1cb721)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4393f1571..8336463664 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.13.0...v18.14.0) (2022-03-24) + + +### Features + +* Add tags to EKS created cluster security group to match rest of module tagging scheme ([#1957](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1957)) ([9371a29](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/9371a2943b13cc2d9ceb34aef14ec2ccee1cb721)) + ## [18.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.12.0...v18.13.0) (2022-03-23) From fd3a3e9a96d9a8fa9b22446e2ac8c36cdf68c5fc Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 24 Mar 2022 10:56:55 -0400 Subject: [PATCH 512/962] fix: Default to cluster version for EKS and self managed node groups when a `cluster_version` is not specified (#1963) --- README.md | 3 ++- examples/complete/main.tf | 10 ++++------ main.tf | 2 +- node_groups.tf | 4 ++-- outputs.tf | 5 +++++ 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d4c7863f4d..c27a829883 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,7 @@ module "karpenter_irsa" { role_name = "karpenter_controller" attach_karpenter_controller_policy = true - karpenter_controller_cluster_ids = [module.eks.cluster_id] + karpenter_controller_cluster_id = module.eks.cluster_id karpenter_controller_node_iam_role_arns = [ module.eks.eks_managed_node_groups["default"].iam_role_arn ] @@ -938,6 +938,7 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [cluster\_version](#output\_cluster\_version) | The Kubernetes version for the cluster | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index bc519d2f3c..961fcb8dcc 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -3,9 +3,8 @@ provider "aws" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.21" - region = "eu-west-1" + name = "ex-${replace(basename(path.cwd), "_", "-")}" + region = "eu-west-1" tags = { Example = local.name @@ -22,7 +21,6 @@ module "eks" { source = "../.." cluster_name = local.name - cluster_version = local.cluster_version cluster_endpoint_private_access = true cluster_endpoint_public_access = true @@ -188,7 +186,7 @@ module "eks_managed_node_group" { name = "separate-eks-mng" cluster_name = module.eks.cluster_id - cluster_version = local.cluster_version + cluster_version = module.eks.cluster_version vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets @@ -205,7 +203,7 @@ module "self_managed_node_group" { name = "separate-self-mng" cluster_name = module.eks.cluster_id - cluster_version = local.cluster_version + cluster_version = module.eks.cluster_version cluster_endpoint = module.eks.cluster_endpoint cluster_auth_base64 = module.eks.cluster_certificate_authority_data diff --git a/main.tf b/main.tf index ad3504ba69..35879fc01b 100644 --- a/main.tf +++ b/main.tf @@ -60,7 +60,7 @@ resource "aws_eks_cluster" "this" { } resource "aws_ec2_tag" "cluster_primary_security_group" { - for_each = { for k, v in merge(var.tags, var.cluster_tags) : k => v if var.create } + for_each = { for k, v in merge(var.tags, var.cluster_tags) : k => v if local.create } resource_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id key = each.key diff --git a/node_groups.tf b/node_groups.tf index f80e444415..76ca3fc411 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -231,7 +231,7 @@ module "eks_managed_node_group" { create = try(each.value.create, true) cluster_name = aws_eks_cluster.this[0].name - cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, var.cluster_version) + cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, aws_eks_cluster.this[0].version) cluster_security_group_id = local.cluster_security_group_id cluster_ip_family = var.cluster_ip_family @@ -402,7 +402,7 @@ module "self_managed_node_group" { ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") - cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, var.cluster_version) + cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, aws_eks_cluster.this[0].version) instance_type = try(each.value.instance_type, var.self_managed_node_group_defaults.instance_type, "m6i.large") key_name = try(each.value.key_name, var.self_managed_node_group_defaults.key_name, null) diff --git a/outputs.tf b/outputs.tf index 64eb97da27..03eede315b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -27,6 +27,11 @@ output "cluster_oidc_issuer_url" { value = try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "") } +output "cluster_version" { + description = "The Kubernetes version for the cluster" + value = try(aws_eks_cluster.this[0].version, "") +} + output "cluster_platform_version" { description = "Platform version for the cluster" value = try(aws_eks_cluster.this[0].platform_version, "") From 7aed57b173e0864a90e5146365041f0a93a9d7cb Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 24 Mar 2022 14:58:20 +0000 Subject: [PATCH 513/962] chore(release): version 18.14.1 [skip ci] ### [18.14.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.14.0...v18.14.1) (2022-03-24) ### Bug Fixes * Default to cluster version for EKS and self managed node groups when a `cluster_version` is not specified ([#1963](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1963)) ([fd3a3e9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/fd3a3e9a96d9a8fa9b22446e2ac8c36cdf68c5fc)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8336463664..e80f177bc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.14.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.14.0...v18.14.1) (2022-03-24) + + +### Bug Fixes + +* Default to cluster version for EKS and self managed node groups when a `cluster_version` is not specified ([#1963](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1963)) ([fd3a3e9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/fd3a3e9a96d9a8fa9b22446e2ac8c36cdf68c5fc)) + ## [18.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.13.0...v18.14.0) (2022-03-24) From 0269d38fcae2b1ca566427159d33910fe96299a7 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 25 Mar 2022 04:26:16 -0400 Subject: [PATCH 514/962] feat: Update TLS provider and remove unnecessary cloud init version requirements (#1966) --- README.md | 4 ++-- examples/eks_managed_node_group/main.tf | 9 +++++---- modules/eks-managed-node-group/README.md | 1 - modules/eks-managed-node-group/versions.tf | 4 ---- modules/self-managed-node-group/README.md | 1 - modules/self-managed-node-group/versions.tf | 4 ---- versions.tf | 2 +- 7 files changed, 8 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c27a829883..1a3409a27c 100644 --- a/README.md +++ b/README.md @@ -807,14 +807,14 @@ Full contributing [guidelines are covered here](https://github.com/terraform-aws |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | -| [tls](#requirement\_tls) | >= 2.2 | +| [tls](#requirement\_tls) | >= 3.0 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | -| [tls](#provider\_tls) | >= 2.2 | +| [tls](#provider\_tls) | >= 3.0 | ## Modules diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 639a56f827..225c5140d0 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -568,10 +568,11 @@ resource "aws_launch_template" "external" { enabled = true } - network_interfaces { - associate_public_ip_address = false - delete_on_termination = true - } + # Disabling due to https://github.com/hashicorp/terraform-provider-aws/issues/23766 + # network_interfaces { + # associate_public_ip_address = false + # delete_on_termination = true + # } # if you want to use a custom AMI # image_id = var.ami_id diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index a89c5f9df0..6c02fe5974 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -50,7 +50,6 @@ module "eks_managed_node_group" { |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | -| [cloudinit](#requirement\_cloudinit) | >= 2.0 | ## Providers diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index e196e178e4..22e8d7265f 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -6,9 +6,5 @@ terraform { source = "hashicorp/aws" version = ">= 3.72" } - cloudinit = { - source = "hashicorp/cloudinit" - version = ">= 2.0" - } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index d42a46ac76..128b8348a2 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -41,7 +41,6 @@ module "self_managed_node_group" { |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | -| [cloudinit](#requirement\_cloudinit) | >= 2.0 | ## Providers diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index e196e178e4..22e8d7265f 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -6,9 +6,5 @@ terraform { source = "hashicorp/aws" version = ">= 3.72" } - cloudinit = { - source = "hashicorp/cloudinit" - version = ">= 2.0" - } } } diff --git a/versions.tf b/versions.tf index cb7e557e9d..85e4ed7915 100644 --- a/versions.tf +++ b/versions.tf @@ -8,7 +8,7 @@ terraform { } tls = { source = "hashicorp/tls" - version = ">= 2.2" + version = ">= 3.0" } } } From 4898bf326d7019fdd9cc33b533a226d6ef3a0f52 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 25 Mar 2022 08:26:54 +0000 Subject: [PATCH 515/962] chore(release): version 18.15.0 [skip ci] ## [18.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.14.1...v18.15.0) (2022-03-25) ### Features * Update TLS provider and remove unnecessary cloud init version requirements ([#1966](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1966)) ([0269d38](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0269d38fcae2b1ca566427159d33910fe96299a7)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e80f177bc5..d39d094d1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.14.1...v18.15.0) (2022-03-25) + + +### Features + +* Update TLS provider and remove unnecessary cloud init version requirements ([#1966](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1966)) ([0269d38](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0269d38fcae2b1ca566427159d33910fe96299a7)) + ### [18.14.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.14.0...v18.14.1) (2022-03-24) From 7a9458af52ddf1f6180324e845b1e8a26fd5c1f5 Mon Sep 17 00:00:00 2001 From: Chris McNabb Date: Tue, 29 Mar 2022 03:07:13 -0400 Subject: [PATCH 516/962] feat: Support default_tags in aws_autoscaling_group (#1973) --- examples/complete/main.tf | 8 ++++++++ modules/self-managed-node-group/README.md | 2 ++ modules/self-managed-node-group/main.tf | 4 +++- modules/self-managed-node-group/variables.tf | 6 ++++++ node_groups.tf | 3 ++- 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 961fcb8dcc..79fb6c018e 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -1,5 +1,11 @@ provider "aws" { region = local.region + + default_tags { + tags = { + ExampleDefaultTag = "ExampleDefaultValue" + } + } } locals { @@ -216,6 +222,8 @@ module "self_managed_node_group" { module.eks.cluster_security_group_id, ] + use_default_tags = true + tags = merge(local.tags, { Separate = "self-managed-node-group" }) } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 128b8348a2..8d5f1bb2c0 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -68,6 +68,7 @@ module "self_managed_node_group" { | [aws_security_group_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_default_tags.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -161,6 +162,7 @@ module "self_managed_node_group" { | [target\_group\_arns](#input\_target\_group\_arns) | A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing | `list(string)` | `[]` | no | | [termination\_policies](#input\_termination\_policies) | A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default` | `list(string)` | `null` | no | | [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update Default Version each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | +| [use\_default\_tags](#input\_use\_default\_tags) | Enables/disables the use of provider default tags in the tag\_specifications of the Auto Scaling group | `bool` | `false` | no | | [use\_mixed\_instances\_policy](#input\_use\_mixed\_instances\_policy) | Determines whether to use a mixed instances policy in the autoscaling group or not | `bool` | `false` | no | | [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix | `bool` | `true` | no | | [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 40dc4f9a3c..b09b601818 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -2,6 +2,8 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} +data "aws_default_tags" "current" {} + data "aws_ami" "eks_default" { count = var.create ? 1 : 0 @@ -385,7 +387,7 @@ resource "aws_autoscaling_group" "this" { "kubernetes.io/cluster/${var.cluster_name}" = "owned" "k8s.io/cluster/${var.cluster_name}" = "owned" }, - var.tags, + var.use_default_tags ? merge(data.aws_default_tags.current.tags, var.tags) : var.tags ) content { diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 7591c77ef6..c261cdcdb5 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -452,6 +452,12 @@ variable "delete_timeout" { default = null } +variable "use_default_tags" { + description = "Enables/disables the use of provider default tags in the tag_specifications of the Auto Scaling group" + type = bool + default = false +} + ################################################################################ # Autoscaling group schedule ################################################################################ diff --git a/node_groups.tf b/node_groups.tf index 76ca3fc411..4d8e0d787a 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -381,7 +381,8 @@ module "self_managed_node_group" { create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, false) schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, null) - delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) + delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) + use_default_tags = try(each.value.use_default_tags, var.self_managed_node_group_defaults.use_default_tags, false) # User data platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, "linux") From 1c7e3164b0ca3f9c3e12508fb28ee52916963dd2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 29 Mar 2022 07:07:50 +0000 Subject: [PATCH 517/962] chore(release): version 18.16.0 [skip ci] ## [18.16.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.15.0...v18.16.0) (2022-03-29) ### Features * Support default_tags in aws_autoscaling_group ([#1973](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1973)) ([7a9458a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7a9458af52ddf1f6180324e845b1e8a26fd5c1f5)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d39d094d1d..d613fee38e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.16.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.15.0...v18.16.0) (2022-03-29) + + +### Features + +* Support default_tags in aws_autoscaling_group ([#1973](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1973)) ([7a9458a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7a9458af52ddf1f6180324e845b1e8a26fd5c1f5)) + ## [18.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.14.1...v18.15.0) (2022-03-25) From 98e137fad990d51a31d86e908ea593e933fc22a9 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 30 Mar 2022 08:34:44 -0400 Subject: [PATCH 518/962] feat: Add back in CloudWatch log group create deny policy to cluster IAM role (#1974) --- main.tf | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/main.tf b/main.tf index 35879fc01b..31568d28ac 100644 --- a/main.tf +++ b/main.tf @@ -219,6 +219,29 @@ resource "aws_iam_role" "this" { permissions_boundary = var.iam_role_permissions_boundary force_detach_policies = true + # https://github.com/terraform-aws-modules/terraform-aws-eks/issues/920 + # Resources running on the cluster are still generaring logs when destroying the module resources + # which results in the log group being re-created even after Terraform destroys it. Removing the + # ability for the cluster role to create the log group prevents this log group from being re-created + # outside of Terraform due to services still generating logs during destroy process + dynamic "inline_policy" { + for_each = var.create_cloudwatch_log_group ? [1] : [] + content { + name = local.iam_role_name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = ["logs:CreateLogGroup"] + Effect = "Deny" + Resource = aws_cloudwatch_log_group.this[0].arn + }, + ] + }) + } + } + tags = merge(var.tags, var.iam_role_tags) } From 6d7245621f97bb8e38642a9e40ddce3a32ff9efb Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 30 Mar 2022 12:35:22 +0000 Subject: [PATCH 519/962] chore(release): version 18.17.0 [skip ci] ## [18.17.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.16.0...v18.17.0) (2022-03-30) ### Features * Add back in CloudWatch log group create deny policy to cluster IAM role ([#1974](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1974)) ([98e137f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/98e137fad990d51a31d86e908ea593e933fc22a9)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d613fee38e..d9629b96e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.17.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.16.0...v18.17.0) (2022-03-30) + + +### Features + +* Add back in CloudWatch log group create deny policy to cluster IAM role ([#1974](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1974)) ([98e137f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/98e137fad990d51a31d86e908ea593e933fc22a9)) + ## [18.16.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.15.0...v18.16.0) (2022-03-29) From 381144e3bb604b3086ceea537a6052a6179ce5b3 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 2 Apr 2022 14:49:25 -0400 Subject: [PATCH 520/962] fix: Correct `capacity_reservation_target` within launch templates of both EKS and self managed node groups (#1979) --- .pre-commit-config.yaml | 2 +- examples/self_managed_node_group/README.md | 1 + examples/self_managed_node_group/main.tf | 14 ++++++++++++++ modules/eks-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/main.tf | 2 +- 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a010fdd41..f00fbfe8aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.64.0 + rev: v1.64.1 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index f91bbf6b4c..3798d8e17e 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -49,6 +49,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| +| [aws_ec2_capacity_reservation.targeted](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ec2_capacity_reservation) | resource | | [aws_key_pair.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | | [aws_kms_key.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 43da796503..5ba1f7f0ad 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -202,6 +202,12 @@ module "eks" { instance_metadata_tags = "disabled" } + capacity_reservation_specification = { + capacity_reservation_target = { + capacity_reservation_id = aws_ec2_capacity_reservation.targeted.id + } + } + create_iam_role = true iam_role_name = "self-managed-node-group-complete-example" iam_role_use_name_prefix = false @@ -407,6 +413,14 @@ resource "aws_kms_key" "ebs" { policy = data.aws_iam_policy_document.ebs.json } +resource "aws_ec2_capacity_reservation" "targeted" { + instance_type = "m6i.large" + instance_platform = "Linux/UNIX" + availability_zone = "${local.region}a" + instance_count = 1 + instance_match_criteria = "targeted" +} + # This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes data "aws_iam_policy_document" "ebs" { # Copy of default KMS policy that lets you manage it diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 662b2aa258..90b300b906 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -94,7 +94,7 @@ resource "aws_launch_template" "this" { capacity_reservation_preference = lookup(capacity_reservation_specification.value, "capacity_reservation_preference", null) dynamic "capacity_reservation_target" { - for_each = lookup(capacity_reservation_specification.value, "capacity_reservation_target", []) + for_each = try([capacity_reservation_specification.value.capacity_reservation_target], []) content { capacity_reservation_id = lookup(capacity_reservation_target.value, "capacity_reservation_id", null) } diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index b09b601818..e625582cd7 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -97,7 +97,7 @@ resource "aws_launch_template" "this" { capacity_reservation_preference = lookup(capacity_reservation_specification.value, "capacity_reservation_preference", null) dynamic "capacity_reservation_target" { - for_each = lookup(capacity_reservation_specification.value, "capacity_reservation_target", []) + for_each = try([capacity_reservation_specification.value.capacity_reservation_target], []) content { capacity_reservation_id = lookup(capacity_reservation_target.value, "capacity_reservation_id", null) } From 62124611d28a90339fa694c12d82974fed932f27 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 2 Apr 2022 18:50:02 +0000 Subject: [PATCH 521/962] chore(release): version 18.17.1 [skip ci] ### [18.17.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.17.0...v18.17.1) (2022-04-02) ### Bug Fixes * Correct `capacity_reservation_target` within launch templates of both EKS and self managed node groups ([#1979](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1979)) ([381144e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/381144e3bb604b3086ceea537a6052a6179ce5b3)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9629b96e5..d613666f1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.17.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.17.0...v18.17.1) (2022-04-02) + + +### Bug Fixes + +* Correct `capacity_reservation_target` within launch templates of both EKS and self managed node groups ([#1979](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1979)) ([381144e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/381144e3bb604b3086ceea537a6052a6179ce5b3)) + ## [18.17.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.16.0...v18.17.0) (2022-03-30) From 523144e1d7d4f64ccf30656078fd10d7cd63a444 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 3 Apr 2022 05:25:42 -0400 Subject: [PATCH 522/962] feat: Add support for allowing EFA network interfaces (#1980) --- examples/self_managed_node_group/main.tf | 31 ++++++++++++++++++++++++ modules/eks-managed-node-group/main.tf | 13 ++++++---- modules/self-managed-node-group/main.tf | 11 ++++++--- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 5ba1f7f0ad..373567b897 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -146,6 +146,37 @@ module "eks" { } } + efa = { + min_size = 1 + max_size = 2 + desired_size = 1 + + # aws ec2 describe-instance-types --region eu-west-1 --filters Name=network-info.efa-supported,Values=true --query "InstanceTypes[*].[InstanceType]" --output text | sort + instance_type = "c5n.9xlarge" + + post_bootstrap_user_data = <<-EOT + + # Install EFA + curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz + tar -xf aws-efa-installer-latest.tar.gz && cd aws-efa-installer + ./efa_installer.sh -y --minimal + fi_info -p efa -t FI_EP_RDM + + # Disable ptrace + sysctl -w kernel.yama.ptrace_scope=0 + EOT + + network_interfaces = [ + { + description = "EFA interface example" + delete_on_termination = true + device_index = 0 + associate_public_ip_address = false + interface_type = "efa" + } + ] + } + # Complete complete = { name = "complete-self-mng" diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 90b300b906..774fcef378 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -38,6 +38,8 @@ locals { use_custom_launch_template = var.create_launch_template || var.launch_template_name != "" launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-eks-node-group") + + security_group_ids = compact(concat([try(aws_security_group.this[0].id, ""), var.cluster_primary_security_group_id], var.vpc_security_group_ids)) } resource "aws_launch_template" "this" { @@ -54,7 +56,7 @@ resource "aws_launch_template" "this" { key_name = var.key_name user_data = module.user_data.user_data - vpc_security_group_ids = compact(concat([try(aws_security_group.this[0].id, ""), var.cluster_primary_security_group_id], var.vpc_security_group_ids)) + vpc_security_group_ids = length(var.network_interfaces) > 0 ? [] : local.security_group_ids default_version = var.launch_template_default_version update_default_version = var.update_launch_template_default_version @@ -208,16 +210,17 @@ resource "aws_launch_template" "this" { delete_on_termination = lookup(network_interfaces.value, "delete_on_termination", null) description = lookup(network_interfaces.value, "description", null) device_index = lookup(network_interfaces.value, "device_index", null) - ipv4_addresses = lookup(network_interfaces.value, "ipv4_addresses", null) != null ? network_interfaces.value.ipv4_addresses : [] + interface_type = lookup(network_interfaces.value, "interface_type", null) + ipv4_addresses = try(network_interfaces.value.ipv4_addresses, []) ipv4_address_count = lookup(network_interfaces.value, "ipv4_address_count", null) - ipv6_addresses = lookup(network_interfaces.value, "ipv6_addresses", null) != null ? network_interfaces.value.ipv6_addresses : [] + ipv6_addresses = try(network_interfaces.value.ipv6_addresses, []) ipv6_address_count = lookup(network_interfaces.value, "ipv6_address_count", null) network_interface_id = lookup(network_interfaces.value, "network_interface_id", null) private_ip_address = lookup(network_interfaces.value, "private_ip_address", null) - security_groups = lookup(network_interfaces.value, "security_groups", null) != null ? network_interfaces.value.security_groups : [] + security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) # Set on EKS managed node group, will fail if set here # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics - # subnet_id = lookup(network_interfaces.value, "subnet_id", null) + # subnet_id = lookup(network_interfaces.value, "subnet_id", null) } } diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index e625582cd7..6278cecf38 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -44,6 +44,8 @@ module "user_data" { locals { launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-node-group") + + security_group_ids = compact(concat([try(aws_security_group.this[0].id, ""), var.cluster_primary_security_group_id], var.vpc_security_group_ids)) } resource "aws_launch_template" "this" { @@ -59,7 +61,7 @@ resource "aws_launch_template" "this" { key_name = var.key_name user_data = module.user_data.user_data - vpc_security_group_ids = compact(concat([try(aws_security_group.this[0].id, ""), var.cluster_primary_security_group_id], var.vpc_security_group_ids)) + vpc_security_group_ids = length(var.network_interfaces) > 0 ? [] : local.security_group_ids default_version = var.launch_template_default_version update_default_version = var.update_launch_template_default_version @@ -203,13 +205,14 @@ resource "aws_launch_template" "this" { delete_on_termination = lookup(network_interfaces.value, "delete_on_termination", null) description = lookup(network_interfaces.value, "description", null) device_index = lookup(network_interfaces.value, "device_index", null) - ipv4_addresses = lookup(network_interfaces.value, "ipv4_addresses", null) != null ? network_interfaces.value.ipv4_addresses : [] + interface_type = lookup(network_interfaces.value, "interface_type", null) + ipv4_addresses = try(network_interfaces.value.ipv4_addresses, []) ipv4_address_count = lookup(network_interfaces.value, "ipv4_address_count", null) - ipv6_addresses = lookup(network_interfaces.value, "ipv6_addresses", null) != null ? network_interfaces.value.ipv6_addresses : [] + ipv6_addresses = try(network_interfaces.value.ipv6_addresses, []) ipv6_address_count = lookup(network_interfaces.value, "ipv6_address_count", null) network_interface_id = lookup(network_interfaces.value, "network_interface_id", null) private_ip_address = lookup(network_interfaces.value, "private_ip_address", null) - security_groups = lookup(network_interfaces.value, "security_groups", null) != null ? network_interfaces.value.security_groups : [] + security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) subnet_id = lookup(network_interfaces.value, "subnet_id", null) } } From f7b4798fcd2ff977da6830425d17628d934df29e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 3 Apr 2022 09:26:21 +0000 Subject: [PATCH 523/962] chore(release): version 18.18.0 [skip ci] ## [18.18.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.17.1...v18.18.0) (2022-04-03) ### Features * Add support for allowing EFA network interfaces ([#1980](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1980)) ([523144e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/523144e1d7d4f64ccf30656078fd10d7cd63a444)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d613666f1e..957083ebca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.18.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.17.1...v18.18.0) (2022-04-03) + + +### Features + +* Add support for allowing EFA network interfaces ([#1980](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1980)) ([523144e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/523144e1d7d4f64ccf30656078fd10d7cd63a444)) + ### [18.17.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.17.0...v18.17.1) (2022-04-02) From 262b480c34c7e38810a15ba027182f4f1c2939cb Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 3 Apr 2022 13:48:06 -0400 Subject: [PATCH 524/962] docs: Re-organize documentation for easier navigation and support for references in issues/PRs (#1981) --- .github/CONTRIBUTING.md | 33 -- README.md | 719 ++---------------------- docs/README.md | 12 + {.github => docs}/UPGRADE-17.0.md | 0 UPGRADE-18.0.md => docs/UPGRADE-18.0.md | 2 + docs/compute_resourcs.md | 209 +++++++ docs/faq.md | 110 ++++ docs/irsa_integration.md | 84 +++ docs/network_connectivity.md | 68 +++ docs/user_data.md | 97 ++++ examples/README.md | 8 + modules/_user_data/README.md | 75 +-- 12 files changed, 639 insertions(+), 778 deletions(-) delete mode 100644 .github/CONTRIBUTING.md create mode 100644 docs/README.md rename {.github => docs}/UPGRADE-17.0.md (100%) rename UPGRADE-18.0.md => docs/UPGRADE-18.0.md (98%) create mode 100644 docs/compute_resourcs.md create mode 100644 docs/faq.md create mode 100644 docs/irsa_integration.md create mode 100644 docs/network_connectivity.md create mode 100644 docs/user_data.md create mode 100644 examples/README.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md deleted file mode 100644 index 53127506fa..0000000000 --- a/.github/CONTRIBUTING.md +++ /dev/null @@ -1,33 +0,0 @@ -# Contributing - -When contributing to this repository, please first discuss the change you wish to make via issue, -email, or any other method with the owners of this repository before making a change. - -Please note we have a code of conduct, please follow it in all your interactions with the project. - -## Pull Request Process - -1. Ensure any install or build dependencies are removed before the end of the layer when doing a build. -2. Update the README.md with details of changes to the interface, this includes new environment variables, exposed ports, useful file locations, and container parameters. -3. Once all outstanding comments and checklist items have been addressed, your contribution will be merged! Merged PRs will be included in the next release. The terraform-aws-eks maintainers take care of updating the CHANGELOG as they merge. - -## Checklists for contributions - -- [ ] Add [semantics prefix](#semantic-pull-requests) to your PR or Commits (at least one of your commit groups) -- [ ] CI tests are passing -- [ ] README.md has been updated after any changes to variables and outputs. See https://github.com/terraform-aws-modules/terraform-aws-eks/#doc-generation - -## Semantic Pull Requests - -To generate changelog, Pull Requests or Commits must have semantic and must follow conventional specs below: - -- `feat:` for new features -- `fix:` for bug fixes -- `improvement:` for enhancements -- `docs:` for documentation and examples -- `refactor:` for code refactoring -- `test:` for tests -- `ci:` for CI purpose -- `chore:` for chores stuff - -The `chore` prefix skipped during changelog generation. It can be used for `chore: update changelog` commit message by example. diff --git a/README.md b/README.md index 1a3409a27c..e1fd75405e 100644 --- a/README.md +++ b/README.md @@ -4,63 +4,64 @@ Terraform module which creates AWS EKS (Kubernetes) resources [![SWUbanner](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner2-direct.svg)](https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md) +## [Documentation](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs) + +- [Frequently Asked Questions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md) +- [Compute Resources](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/compute_resources.md) +- [IRSA Integration](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/irsa-integration.md) +- [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/user_data.md) +- [Network Connectivity](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/network_connectivity.md) +- Upgrade Guides + - [Upgrade to v17.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-17.0.md) + - [Upgrade to v18.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-18.0.md) + +### External Documentation + +Please note that we strive to provide a comprehensive suite of documentation for __*configuring and utilizing the module(s)*__ defined here, and that documentation regarding EKS (including EKS managed node group, self managed node group, and Fargate profile) and/or Kubernetes features, usage, etc. are better left up to their respective sources: +- [AWS EKS Documentation](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) +- [Kubernetes Documentation](https://kubernetes.io/docs/home/) + ## Available Features -- AWS EKS Cluster - AWS EKS Cluster Addons - AWS EKS Identity Provider Configuration - All [node types](https://docs.aws.amazon.com/eks/latest/userguide/eks-compute.html) are supported: - [EKS Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) - [Self Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) - [Fargate Profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) -- Support for custom AMI, custom launch template, and custom user data +- Support for custom AMI, custom launch template, and custom user data including custom user data template - Support for Amazon Linux 2 EKS Optimized AMI and Bottlerocket nodes - Windows based node support is limited to a default user data template that is provided due to the lack of Windows support and manual steps required to provision Windows based EKS nodes - Support for module created security group, bring your own security groups, as well as adding additional security group rules to the module created security group(s) -- Support for providing maps of node groups/Fargate profiles to the cluster module definition or use separate node group/Fargate profile sub-modules -- Provisions to provide node group/Fargate profile "default" settings - useful for when creating multiple node groups/Fargate profiles where you want to set a common set of configurations once, and then individual control only select features - -### ℹ️ `Error: Invalid for_each argument ...` +- Support for creating node groups/profiles separate from the cluster through the use of sub-modules (same as what is used by root module) +- Support for node group/profile "default" settings - useful for when creating multiple node groups/Fargate profiles where you want to set a common set of configurations once, and then individually control only select features on certain node groups/profiles -Users may encounter an error such as `Error: Invalid for_each argument - The "for_each" value depends on resource attributes that cannot be determined until apply, so Terraform cannot predict how many instances will be created. To work around this, use the -target argument to first apply ...` +### [IRSA Terraform Module](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) -This error is due to an upstream issue with [Terraform core](https://github.com/hashicorp/terraform/issues/4149). There are two potential options you can take to help mitigate this issue: +An IAM role for service accounts (IRSA) sub-module has been created to make deploying common addons/controllers easier. Instead of users having to create a custom IAM role with the necessary federated role assumption required for IRSA plus find and craft the associated policy required for the addon/controller, users can create the IRSA role and policy with a few lines of code. See the [`terraform-aws-iam/examples/iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/examples/iam-role-for-service-accounts-eks/main.tf) directory for examples on how to use the IRSA sub-module in conjunction with this (`terraform-aws-eks`) module. -1. Create the dependent resources before the cluster => `terraform apply -target ` and then `terraform apply` for the cluster (or other similar means to just ensure the referenced resources exist before creating the cluster) - - Note: this is the route users will have to take for adding additional security groups to nodes since there isn't a separate "security group attachment" resource -2. For additional IAM policies, users can attach the policies outside of the cluster definition as demonstrated below +Some of the addon/controller policies that are currently supported include: -```hcl -resource "aws_iam_role_policy_attachment" "additional" { - for_each = module.eks.eks_managed_node_groups - # you could also do the following or any combination: - # for_each = merge( - # module.eks.eks_managed_node_groups, - # module.eks.self_managed_node_group, - # module.eks.fargate_profile, - # ) - - # This policy does not have to exist at the time of cluster creation. Terraform can - # deduce the proper order of its creation to avoid errors during creation - policy_arn = aws_iam_policy.node_additional.arn - role = each.value.iam_role_name -} -``` +- [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md) +- [External DNS](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#iam-policy) +- [EBS CSI Driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/example-iam-policy.json) +- [VPC CNI](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html) +- [Node Termination Handler](https://github.com/aws/aws-node-termination-handler#5-create-an-iam-role-for-the-pods) +- [Karpenter](https://github.com/aws/karpenter/blob/main/website/content/en/preview/getting-started/cloudformation.yaml) +- [Load Balancer Controller](https://github.com/kubernetes-sigs/aws-load-balancer-controller/blob/main/docs/install/iam_policy.json) -The tl;dr for this issue is that the Terraform resource passed into the modules map definition *must* be known before you can apply the EKS module. The variables this potentially affects are: - -- `cluster_security_group_additional_rules` (i.e. - referencing an external security group resource in a rule) -- `node_security_group_additional_rules` (i.e. - referencing an external security group resource in a rule) -- `iam_role_additional_policies` (i.e. - referencing an external policy resource) +See [terraform-aws-iam/modules/iam-role-for-service-accounts](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) for current list of supported addon/controller policies as more are added to the project. ## Usage ```hcl module "eks" { - source = "terraform-aws-modules/eks/aws" + source = "terraform-aws-modules/eks/aws" + version = "~> 18.0" + + cluster_name = "my-cluster" + cluster_version = "1.21" - cluster_name = "my-cluster" - cluster_version = "1.21" cluster_endpoint_private_access = true cluster_endpoint_public_access = true @@ -86,14 +87,14 @@ module "eks" { self_managed_node_group_defaults = { instance_type = "m6i.large" update_launch_template_default_version = true - iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"] + iam_role_additional_policies = [ + "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + ] } self_managed_node_groups = { one = { - name = "spot-1" - - public_ip = true + name = "mixed-1" max_size = 5 desired_size = 2 @@ -116,29 +117,13 @@ module "eks" { }, ] } - - pre_bootstrap_user_data = <<-EOT - echo "foo" - export FOO=bar - EOT - - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" - - post_bootstrap_user_data = <<-EOT - cd /tmp - sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm - sudo systemctl enable amazon-ssm-agent - sudo systemctl start amazon-ssm-agent - EOT } } # EKS Managed Node Group(s) eks_managed_node_group_defaults = { - ami_type = "AL2_x86_64" - disk_size = 50 - instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] - vpc_security_group_ids = [aws_security_group.additional.id] + disk_size = 50 + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] } eks_managed_node_groups = { @@ -150,21 +135,6 @@ module "eks" { instance_types = ["t3.large"] capacity_type = "SPOT" - labels = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - taints = { - dedicated = { - key = "dedicated" - value = "gpuGroup" - effect = "NO_SCHEDULE" - } - } - tags = { - ExtraTag = "example" - } } } @@ -173,108 +143,10 @@ module "eks" { default = { name = "default" selectors = [ - { - namespace = "kube-system" - labels = { - k8s-app = "kube-dns" - } - }, { namespace = "default" } ] - - tags = { - Owner = "test" - } - - timeouts = { - create = "20m" - delete = "20m" - } - } - } - - tags = { - Environment = "dev" - Terraform = "true" - } -} -``` - -### IRSA Integration - -An [IAM role for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) module has been created to work in conjunction with the EKS module. The [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) module has a set of pre-defined IAM policies for common addons/controllers/custom resources to allow users to quickly enable common integrations. Check [`policy.tf`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/modules/iam-role-for-service-accounts-eks/policies.tf) for a list of the policies currently supported. A example of this integration is shown below, and more can be found in the [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/examples/iam-role-for-service-accounts-eks/main.tf) example directory: - -```hcl -module "eks" { - source = "terraform-aws-modules/eks/aws" - - cluster_name = "example" - cluster_version = "1.21" - - cluster_addons = { - vpc-cni = { - resolve_conflicts = "OVERWRITE" - service_account_role_arn = module.vpc_cni_irsa.iam_role_arn - } - } - - vpc_id = "vpc-1234556abcdef" - subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] - - eks_managed_node_group_defaults = { - # We are using the IRSA created below for permissions - # This is a better practice as well so that the nodes do not have the permission, - # only the VPC CNI addon will have the permission - iam_role_attach_cni_policy = false - } - - eks_managed_node_groups = { - default = {} - } - - tags = { - Environment = "dev" - Terraform = "true" - } -} - -module "vpc_cni_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - - role_name = "vpc_cni" - attach_vpc_cni_policy = true - vpc_cni_enable_ipv4 = true - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:aws-node"] - } - } - - tags = { - Environment = "dev" - Terraform = "true" - } -} - -module "karpenter_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - - role_name = "karpenter_controller" - attach_karpenter_controller_policy = true - - karpenter_controller_cluster_id = module.eks.cluster_id - karpenter_controller_node_iam_role_arns = [ - module.eks.eks_managed_node_groups["default"].iam_role_arn - ] - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["karpenter:karpenter"] } } @@ -285,507 +157,6 @@ module "karpenter_irsa" { } ``` -## Node Group Configuration - -⚠️ The configurations shown below are referenced from within the root EKS module; there will be slight differences in the default values provided when compared to the underlying sub-modules (`eks-managed-node-group`, `self-managed-node-group`, and `fargate-profile`). - -### EKS Managed Node Groups - -ℹ️ Only the pertinent attributes are shown for brevity - -1. AWS EKS Managed Node Group can provide its own launch template and utilize the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version. By default, the module creates a launch template to ensure tags are propagated to instances, etc., so we need to disable it to use the default template provided by the AWS EKS managed node group service: - -```hcl - eks_managed_node_groups = { - default = { - create_launch_template = false - launch_template_name = "" - } - } -``` - -2. AWS EKS Managed Node Group also offers native, default support for Bottlerocket OS by simply specifying the AMI type: - -```hcl - eks_managed_node_groups = { - bottlerocket_default = { - create_launch_template = false - launch_template_name = "" - - ami_type = "BOTTLEROCKET_x86_64" - platform = "bottlerocket" - } - } -``` - -3. AWS EKS Managed Node Groups allow you to extend configurations by providing your own launch template and user data that is merged with what the service provides. For example, to provide additional user data before the nodes are bootstrapped as well as supply additional arguments to the bootstrap script: - -```hcl - eks_managed_node_groups = { - extend_config = { - # This is supplied to the AWS EKS Optimized AMI - # bootstrap script https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh - bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" - - # This user data will be injected prior to the user data provided by the - # AWS EKS Managed Node Group service (contains the actually bootstrap configuration) - pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - EOT - } - } -``` - -4. The same configurations extension is offered when utilizing Bottlerocket OS AMIs, but the user data is slightly different. Bottlerocket OS uses a TOML user data file and you can provide additional configuration settings via the `bootstrap_extra_args` variable which gets merged into what is provided by the AWS EKS Managed Node Service: - -```hcl - eks_managed_node_groups = { - bottlerocket_extend_config = { - ami_type = "BOTTLEROCKET_x86_64" - platform = "bottlerocket" - - # this will get added to what AWS provides - bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" - EOT - } - } -``` - -5. Users can also utilize a custom AMI, but doing so means that AWS EKS Managed Node Group will NOT inject the necessary bootstrap script and configurations into the user data supplied to the launch template. When using a custom AMI, users must also opt in to bootstrapping the nodes via user data and either use the module default user data template or provide your own user data template file: - -```hcl - eks_managed_node_groups = { - custom_ami = { - ami_id = "ami-0caf35bc73450c396" - - # By default, EKS managed node groups will not append bootstrap script; - # this adds it back in using the default template provided by the module - # Note: this assumes the AMI provided is an EKS optimized AMI derivative - enable_bootstrap_user_data = true - - bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" - - pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - EOT - - # Because we have full control over the user data supplied, we can also run additional - # scripts/configuration changes after the bootstrap script has been run - post_bootstrap_user_data = <<-EOT - echo "you are free little kubelet!" - EOT - } - } -``` - -6. Similarly, for Bottlerocket there is similar support: - -```hcl - eks_managed_node_groups = { - bottlerocket_custom_ami = { - ami_id = "ami-0ff61e0bcfc81dc94" - platform = "bottlerocket" - - # use module user data template to bootstrap - enable_bootstrap_user_data = true - # this will get added to the template - bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" - - [settings.kubernetes.node-labels] - "label1" = "foo" - "label2" = "bar" - - [settings.kubernetes.node-taints] - "dedicated" = "experimental:PreferNoSchedule" - "special" = "true:NoSchedule" - EOT - } - } -``` - -See the [`examples/eks_managed_node_group/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group) for a working example of these configurations. - -### Self Managed Node Groups - -ℹ️ Only the pertinent attributes are shown for brevity - -1. By default, the `self-managed-node-group` sub-module will use the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version: - -```hcl - cluster_version = "1.21" - - # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.21 - self_managed_node_groups = { - default = {} - } -``` - -2. To use Bottlerocket, specify the `platform` as `bottlerocket` and supply the Bottlerocket AMI. The module provided user data for Bottlerocket will be used to bootstrap the nodes created: - -```hcl - cluster_version = "1.21" - - self_managed_node_groups = { - bottlerocket = { - platform = "bottlerocket" - ami_id = data.aws_ami.bottlerocket_ami.id - } - } -``` - -### Fargate Profiles - -Fargate profiles are rather straightforward. Simply supply the necessary information for the desired profile(s). See the [`examples/fargate_profile/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile) for a working example of the various configurations. - -### Mixed Node Groups - -ℹ️ Only the pertinent attributes are shown for brevity - -Users are free to mix and match the different node group types that meet their needs. For example, the following are just an example of the different possibilities: -- AWS EKS Cluster with one or more AWS EKS Managed Node Groups -- AWS EKS Cluster with one or more Self Managed Node Groups -- AWS EKS Cluster with one or more Fargate profiles -- AWS EKS Cluster with one or more AWS EKS Managed Node Groups, one or more Self Managed Node Groups, one or more Fargate profiles - -It is also possible to configure the various node groups of each family differently. Node groups may also be defined outside of the root `eks` module definition by using the provided sub-modules. There are no restrictions on the the various different possibilities provided by the module. - -```hcl - self_managed_node_group_defaults = { - vpc_security_group_ids = [aws_security_group.additional.id] - iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"] - } - - self_managed_node_groups = { - one = { - name = "spot-1" - - public_ip = true - max_size = 5 - desired_size = 2 - - use_mixed_instances_policy = true - mixed_instances_policy = { - instances_distribution = { - on_demand_base_capacity = 0 - on_demand_percentage_above_base_capacity = 10 - spot_allocation_strategy = "capacity-optimized" - } - - override = [ - { - instance_type = "m5.large" - weighted_capacity = "1" - }, - { - instance_type = "m6i.large" - weighted_capacity = "2" - }, - ] - } - - pre_bootstrap_user_data = <<-EOT - echo "foo" - export FOO=bar - EOT - - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" - - post_bootstrap_user_data = <<-EOT - cd /tmp - sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm - sudo systemctl enable amazon-ssm-agent - sudo systemctl start amazon-ssm-agent - EOT - } - } - - # EKS Managed Node Group(s) - eks_managed_node_group_defaults = { - ami_type = "AL2_x86_64" - disk_size = 50 - instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] - vpc_security_group_ids = [aws_security_group.additional.id] - } - - eks_managed_node_groups = { - blue = {} - green = { - min_size = 1 - max_size = 10 - desired_size = 1 - - instance_types = ["t3.large"] - capacity_type = "SPOT" - labels = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - - taints = { - dedicated = { - key = "dedicated" - value = "gpuGroup" - effect = "NO_SCHEDULE" - } - } - - update_config = { - max_unavailable_percentage = 50 # or set `max_unavailable` - } - - tags = { - ExtraTag = "example" - } - } - } - - # Fargate Profile(s) - fargate_profiles = { - default = { - name = "default" - selectors = [ - { - namespace = "kube-system" - labels = { - k8s-app = "kube-dns" - } - }, - { - namespace = "default" - } - ] - - tags = { - Owner = "test" - } - - timeouts = { - create = "20m" - delete = "20m" - } - } - } -``` - -See the [`examples/complete/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete) for a working example of these configurations. - -### Default configurations - -Each node group type (EKS managed node group, self managed node group, or Fargate profile) provides a default configuration setting that allows users to provide their own default configuration instead of the module's default configuration. This allows users to set a common set of defaults for their node groups and still maintain the ability to override these settings within the specific node group definition. The order of precedence for each node group type roughly follows (from highest to least precedence): -- Node group individual configuration - - Node group family default configuration - - Module default configuration - -These are provided via the following variables for the respective node group family: -- `eks_managed_node_group_defaults` -- `self_managed_node_group_defaults` -- `fargate_profile_defaults` - -For example, the following creates 4 AWS EKS Managed Node Groups: - -```hcl - eks_managed_node_group_defaults = { - ami_type = "AL2_x86_64" - disk_size = 50 - instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] - } - - eks_managed_node_groups = { - # Uses defaults provided by module with the default settings above overriding the module defaults - default = {} - - # This further overrides the instance types used - compute = { - instance_types = ["c5.large", "c6i.large", "c6d.large"] - } - - # This further overrides the instance types and disk size used - persistent = { - disk_size = 1024 - instance_types = ["r5.xlarge", "r6i.xlarge", "r5b.xlarge"] - } - - # This overrides the OS used - bottlerocket = { - ami_type = "BOTTLEROCKET_x86_64" - platform = "bottlerocket" - } - } -``` - -## Module Design Considerations - -### General Notes - -While the module is designed to be flexible and support as many use cases and configurations as possible, there is a limit to what first class support can be provided without over-burdening the complexity of the module. Below are a list of general notes on the design intent captured by this module which hopefully explains some of the decisions that are, or will be made, in terms of what is added/supported natively by the module: - -- Despite the addition of Windows Subsystem for Linux (WSL for short), containerization technology is very much a suite of Linux constructs and therefore Linux is the primary OS supported by this module. In addition, due to the first class support provided by AWS, Bottlerocket OS and Fargate Profiles are also very much fully supported by this module. This module does not make any attempt to NOT support Windows, as in preventing the usage of Windows based nodes, however it is up to users to put in additional effort in order to operate Windows based nodes when using the module. User can refer to the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html) for further details. What this means is: - - AWS EKS Managed Node Groups default to `linux` as the `platform`, but `bottlerocket` is also supported by AWS (`windows` is not supported by AWS EKS Managed Node groups) - - AWS Self Managed Node Groups also default to `linux` and the default AMI used is the latest AMI for the selected Kubernetes version. If you wish to use a different OS or AMI then you will need to opt in to the necessary configurations to ensure the correct AMI is used in conjunction with the necessary user data to ensure the nodes are launched and joined to your cluster successfully. -- AWS EKS Managed Node groups are currently the preferred route over Self Managed Node Groups for compute nodes. Both operate very similarly - both are backed by autoscaling groups and launch templates deployed and visible within your account. However, AWS EKS Managed Node groups provide a better user experience and offer a more "managed service" experience and therefore has precedence over Self Managed Node Groups. That said, there are currently inherent limitations as AWS continues to rollout additional feature support similar to the level of customization you can achieve with Self Managed Node Groups. When requesting added feature support for AWS EKS Managed Node groups, please ensure you have verified that the feature(s) are 1) supported by AWS and 2) supported by the Terraform AWS provider before submitting a feature request. -- Due to the plethora of tooling and different manners of configuring your cluster, cluster configuration is intentionally left out of the module in order to simplify the module for a broader user base. Previous module versions provided support for managing the aws-auth configmap via the Kubernetes Terraform provider using the now deprecated aws-iam-authenticator; these are no longer included in the module. This module strictly focuses on the infrastructure resources to provision an EKS cluster as well as any supporting AWS resources. How the internals of the cluster are configured and managed is up to users and is outside the scope of this module. There is an output attribute, `aws_auth_configmap_yaml`, that has been provided that can be useful to help bridge this transition. Please see the various examples provided where this attribute is used to ensure that self managed node groups or external node groups have their IAM roles appropriately mapped to the aws-auth configmap. How users elect to manage the aws-auth configmap is left up to their choosing. - -### User Data & Bootstrapping - -There are a multitude of different possible configurations for how module users require their user data to be configured. In order to better support the various combinations from simple, out of the box support provided by the module to full customization of the user data using a template provided by users - the user data has been abstracted out to its own module. Users can see the various methods of using and providing user data through the [user data examples](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) as well more detailed information on the design and possible configurations via the [user data module itself](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/_user_data) - -In general (tl;dr): -- AWS EKS Managed Node Groups - - `linux` platform (default) -> user data is pre-pended to the AWS provided bootstrap user data (bash/shell script) when using the AWS EKS provided AMI, otherwise users need to opt in via `enable_bootstrap_user_data` and use the module provided user data template or provide their own user data template to bootstrap nodes to join the cluster - - `bottlerocket` platform -> user data is merged with the AWS provided bootstrap user data (TOML file) when using the AWS EKS provided AMI, otherwise users need to opt in via `enable_bootstrap_user_data` and use the module provided user data template or provide their own user data template to bootstrap nodes to join the cluster -- Self Managed Node Groups - - `linux` platform (default) -> the user data template (bash/shell script) provided by the module is used as the default; users are able to provide their own user data template - - `bottlerocket` platform -> the user data template (TOML file) provided by the module is used as the default; users are able to provide their own user data template - - `windows` platform -> the user data template (powershell/PS1 script) provided by the module is used as the default; users are able to provide their own user data template - -Module provided default templates can be found under the [templates directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/templates) - -### Security Groups - -- Cluster Security Group - - This module by default creates a cluster security group ("additional" security group when viewed from the console) in addition to the default security group created by the AWS EKS service. This "additional" security group allows users to customize inbound and outbound rules via the module as they see fit - - The default inbound/outbound rules provided by the module are derived from the [AWS minimum recommendations](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) in addition to NTP and HTTPS public internet egress rules (without, these show up in VPC flow logs as rejects - they are used for clock sync and downloading necessary packages/updates) - - The minimum inbound/outbound rules are provided for cluster and node creation to succeed without errors, but users will most likely need to add the necessary port and protocol for node-to-node communication (this is user specific based on how nodes are configured to communicate across the cluster) - - Users have the ability to opt out of the security group creation and instead provide their own externally created security group if so desired - - The security group that is created is designed to handle the bare minimum communication necessary between the control plane and the nodes, as well as any external egress to allow the cluster to successfully launch without error - - Users also have the option to supply additional, externally created security groups to the cluster as well via the `cluster_additional_security_group_ids` variable - - Lastly, users are able to opt in to attaching the primary security group automatically created by the EKS service by setting `attach_cluster_primary_security_group` = `true` from the root module for the respective node group (or set it within the node group defaults). This security group is not managed by the module; it is created by the EKS service. It permits all traffic within the domain of the security group as well as all egress traffic to the internet. - -- Node Group Security Group(s) - - Each node group (EKS Managed Node Group and Self Managed Node Group) by default creates its own security group. By default, this security group does not contain any additional security group rules. It is merely an "empty container" that offers users the ability to opt into any addition inbound our outbound rules as necessary - - Users also have the option to supply their own, and/or additional, externally created security group(s) to the node group as well via the `vpc_security_group_ids` variable - -See the example snippet below which adds additional security group rules to the cluster security group as well as the shared node security group (for node-to-node access). Users can use this extensibility to open up network access as they see fit using the security groups provided by the module: - -```hcl - ... - # Extend cluster security group rules - cluster_security_group_additional_rules = { - egress_nodes_ephemeral_ports_tcp = { - description = "To node 1025-65535" - protocol = "tcp" - from_port = 1025 - to_port = 65535 - type = "egress" - source_node_security_group = true - } - } - - # Extend node-to-node security group rules - node_security_group_additional_rules = { - ingress_self_all = { - description = "Node to node all ports/protocols" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "ingress" - self = true - } - egress_all = { - description = "Node all egress" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "egress" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] - } - } - ... -``` -The security groups created by this module are depicted in the image shown below along with their default inbound/outbound rules: - -

- Security Groups -

- -## Notes - -- Setting `instance_refresh_enabled = true` will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. See the [instance_refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa_autoscale_refresh) example provided. - -
Frequently Asked Questions
- -

Why are nodes not being registered?

- -Often an issue caused by one of two reasons: -1. Networking or endpoint mis-configuration. -2. Permissions (IAM/RBAC) - -At least one of the cluster public or private endpoints must be enabled to access the cluster to work. If you require a public endpoint, setting up both (public and private) and restricting the public endpoint via setting `cluster_endpoint_public_access_cidrs` is recommended. More info regarding communication with an endpoint is available [here](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html). - -Nodes need to be able to contact the EKS cluster endpoint. By default, the module only creates a public endpoint. To access the endpoint, the nodes need outgoing internet access: - -- Nodes in private subnets: via a NAT gateway or instance along with the appropriate routing rules -- Nodes in public subnets: ensure that nodes are launched with public IPs is enabled (either through the module here or your subnet setting defaults) - -Important: If you apply only the public endpoint and configure the `cluster_endpoint_public_access_cidrs` to restrict access, know that EKS nodes will also use the public endpoint and you must allow access to the endpoint. If not, then your nodes will fail to work correctly. - -Cluster private endpoint can also be enabled by setting `cluster_endpoint_private_access = true` on this module. Node communication to the endpoint stays within the VPC. Ensure that VPC DNS resolution and hostnames are also enabled for your VPC when the private endpoint is enabled. - -Nodes need to be able to connect to other AWS services plus pull down container images from container registries (ECR). If for some reason you cannot enable public internet access for nodes you can add VPC endpoints to the relevant services: EC2 API, ECR API, ECR DKR and S3. - -

How can I work with the cluster if I disable the public endpoint?

- -You have to interact with the cluster from within the VPC that it is associated with; either through a VPN connection, a bastion EC2 instance, etc. - -

How can I stop Terraform from removing the EKS tags from my VPC and subnets?

- -You need to add the tags to the Terraform definition of the VPC and subnets yourself. See the [basic example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/basic). - -An alternative is to use the aws provider's [`ignore_tags` variable](https://www.terraform.io/docs/providers/aws/#ignore_tags-configuration-block). However this can also cause terraform to display a perpetual difference. - -

Why are there no changes when a node group's desired count is modified?

- -The module is configured to ignore this value. Unfortunately, Terraform does not support variables within the `lifecycle` block. The setting is ignored to allow the cluster autoscaler to work correctly so that `terraform apply` does not accidentally remove running workers. You can change the desired count via the CLI or console if you're not using the cluster autoscaler. - -If you are not using autoscaling and want to control the number of nodes via terraform, set the `min_size` and `max_size` for node groups. Before changing those values, you must satisfy AWS `desired_size` constraints (which must be between new min/max values). - -

Why are nodes not recreated when the `launch_template` is recreated?

- -By default the ASG for a self-managed node group is not configured to be recreated when the launch configuration or template changes; you will need to use a process to drain and cycle the nodes. - -If you are NOT using the cluster autoscaler: - -- Add a new instance -- Drain an old node `kubectl drain --force --ignore-daemonsets --delete-local-data ip-xxxxxxx.eu-west-1.compute.internal` -- Wait for pods to be Running -- Terminate the old node instance. ASG will start a new instance -- Repeat the drain and delete process until all old nodes are replaced - -If you are using the cluster autoscaler: - -- Drain an old node `kubectl drain --force --ignore-daemonsets --delete-local-data ip-xxxxxxx.eu-west-1.compute.internal` -- Wait for pods to be Running -- Cluster autoscaler will create new nodes when required -- Repeat until all old nodes are drained -- Cluster autoscaler will terminate the old nodes after 10-60 minutes automatically - -You can also use a third-party tool like Gruntwork's kubergrunt. See the [`eks deploy`](https://github.com/gruntwork-io/kubergrunt#deploy) subcommand. - -Alternatively, use a managed node group instead. -

How can I use Windows workers?

- -To enable Windows support for your EKS cluster, you should apply some configuration manually. See the [Enabling Windows Support (Windows/MacOS/Linux)](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support). - -Windows based nodes require an additional cluster role (`eks:kube-proxy-windows`). - -

Worker nodes with labels do not join a 1.16+ cluster

- -As of Kubernetes 1.16, kubelet restricts which labels with names in the `kubernetes.io` namespace can be applied to nodes. Labels such as `kubernetes.io/lifecycle=spot` are no longer allowed; instead use `node.kubernetes.io/lifecycle=spot` - -See your Kubernetes version's documentation for the `--node-labels` kubelet flag for the allowed prefixes. [Documentation for 1.16](https://v1-16.docs.kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) - -
- ## Examples - [Complete](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete): EKS Cluster using all available node group types in various combinations demonstrating many of the supported features and configurations @@ -797,8 +168,10 @@ See your Kubernetes version's documentation for the `--node-labels` kubelet fla ## Contributing -Report issues/questions/feature requests via [issues](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/new) -Full contributing [guidelines are covered here](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/.github/CONTRIBUTING.md) +We are grateful to the community for contributing bugfixes and improvements! Please see below to learn how you can take part. + +- [Code of Conduct](https://github.com/terraform-aws-modules/.github/blob/master/CODE_OF_CONDUCT.md) +- [Contributing Guide](https://github.com/terraform-aws-modules/.github/blob/master/CONTRIBUTING.md) ## Requirements diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..ccbb3f8ee7 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,12 @@ +# Documentation + +## Table of Contents + +- [Frequently Asked Questions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md) +- [Compute Resources](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/compute_resources.md) +- [IRSA Integration](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/irsa-integration.md) +- [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/user_data.md) +- [Network Connectivity](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/network_connectivity.md) +- Upgrade Guides + - [Upgrade to v17.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-17.0.md) + - [Upgrade to v18.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-18.0.md) diff --git a/.github/UPGRADE-17.0.md b/docs/UPGRADE-17.0.md similarity index 100% rename from .github/UPGRADE-17.0.md rename to docs/UPGRADE-17.0.md diff --git a/UPGRADE-18.0.md b/docs/UPGRADE-18.0.md similarity index 98% rename from UPGRADE-18.0.md rename to docs/UPGRADE-18.0.md index 2d00e6fa98..29ae3ebcd7 100644 --- a/UPGRADE-18.0.md +++ b/docs/UPGRADE-18.0.md @@ -2,6 +2,8 @@ Please consult the `examples` directory for reference example configurations. If you find a bug, please open an issue with supporting configuration to reproduce. +Note: please see https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1744 where users have shared their steps/information for their individual configurations. Due to the numerous configuration possibilities, it is difficult to capture specific steps that will work for all and this has been a very helpful issue for others to share they were able to upgrade. + ## List of backwards incompatible changes - Launch configuration support has been removed and only launch template is supported going forward. AWS is no longer adding new features back into launch configuration and their docs state [`We strongly recommend that you do not use launch configurations. They do not provide full functionality for Amazon EC2 Auto Scaling or Amazon EC2. We provide information about launch configurations for customers who have not yet migrated from launch configurations to launch templates.`](https://docs.aws.amazon.com/autoscaling/ec2/userguide/LaunchConfiguration.html) diff --git a/docs/compute_resourcs.md b/docs/compute_resourcs.md new file mode 100644 index 0000000000..556a2fc84f --- /dev/null +++ b/docs/compute_resourcs.md @@ -0,0 +1,209 @@ +# Compute Resources + +## Table of Contents + +- [EKS Managed Node Groups](https://github.com/terraform-aws-module/terraform-aws-eks/blob/master/docs/node_groups.md#eks-managed-node-groups) +- [Self Managed Node Groups](https://github.com/terraform-aws-module/terraform-aws-eks/blob/master/docs/node_groups.md#self-managed-node-groups) +- [Fargate Profiles](https://github.com/terraform-aws-module/terraform-aws-eks/blob/master/docs/node_groups.md#fargate-profiles) +- [Default Configurations](https://github.com/terraform-aws-module/terraform-aws-eks/blob/master/docs/node_groups.md#default-configurations) + +ℹ️ Only the pertinent attributes are shown below for brevity + +### EKS Managed Node Groups + +Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) documentation for service related details. + +1. The module creates a custom launch template by default to ensure settings such as tags are propagated to instances. To use the default template provided by the AWS EKS managed node group service, disable the launch template creation and set the `launch_template_name` to an empty string: + +```hcl + eks_managed_node_groups = { + default = { + create_launch_template = false + launch_template_name = "" + } + } +``` + +2. Native support for Bottlerocket OS is provided by providing the respective AMI type: + +```hcl + eks_managed_node_groups = { + bottlerocket_default = { + create_launch_template = false + launch_template_name = "" + + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + } + } +``` + +3. Users have limited support to extend the user data that is pre-pended to the user data provided by the AWS EKS Managed Node Group service: + +```hcl + eks_managed_node_groups = { + prepend_userdata = { + # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 + pre_bootstrap_user_data = <<-EOT + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + export KUBELET_EXTRA_ARGS="--max-pods=110" + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + EOT + } + } +``` + +4. Bottlerocket OS is supported in a similar manner. However, note that the user data for Bottlerocket OS uses the TOML format: + +```hcl + eks_managed_node_groups = { + bottlerocket_prepend_userdata = { + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + EOT + } + } +``` + +5. When using a custom AMI, the AWS EKS Managed Node Group service will NOT inject the necessary bootstrap script into the supplied user data. Users can elect to provide their own user data to bootstrap and connect or opt in to use the module provided user data: + +```hcl + eks_managed_node_groups = { + custom_ami = { + ami_id = "ami-0caf35bc73450c396" + + # By default, EKS managed node groups will not append bootstrap script; + # this adds it back in using the default template provided by the module + # Note: this assumes the AMI provided is an EKS optimized AMI derivative + enable_bootstrap_user_data = true + + bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" + + pre_bootstrap_user_data = <<-EOT + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + EOT + + # Because we have full control over the user data supplied, we can also run additional + # scripts/configuration changes after the bootstrap script has been run + post_bootstrap_user_data = <<-EOT + echo "you are free little kubelet!" + EOT + } + } +``` + +6. There is similar support for Bottlerocket OS: + +```hcl + eks_managed_node_groups = { + bottlerocket_custom_ami = { + ami_id = "ami-0ff61e0bcfc81dc94" + platform = "bottlerocket" + + # use module user data template to bootstrap + enable_bootstrap_user_data = true + # this will get added to the template + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + + [settings.kubernetes.node-labels] + "label1" = "foo" + "label2" = "bar" + + [settings.kubernetes.node-taints] + "dedicated" = "experimental:PreferNoSchedule" + "special" = "true:NoSchedule" + EOT + } + } +``` + +See the [`examples/eks_managed_node_group/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group) for a working example of various configurations. + +### Self Managed Node Groups + +Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) documentation for service related details. + +1. The `self-managed-node-group` uses the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version by default: + +```hcl + cluster_version = "1.21" + + # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.21 + self_managed_node_groups = { + default = {} + } +``` + +2. To use Bottlerocket, specify the `platform` as `bottlerocket` and supply a Bottlerocket OS AMI: + +```hcl + cluster_version = "1.21" + + self_managed_node_groups = { + bottlerocket = { + platform = "bottlerocket" + ami_id = data.aws_ami.bottlerocket_ami.id + } + } +``` + +See the [`examples/self_managed_node_group/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self_managed_node_group) for a working example of various configurations. + +### Fargate Profiles + +Fargate profiles are straightforward to use and therefore no further details are provided here. See the [`examples/fargate_profile/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile) for a working example of various configurations. + +### Default Configurations + +Each type of compute resource (EKS managed node group, self managed node group, or Fargate profile) provides the option for users to specify a default configuration. These default configurations can be overridden from within the compute resource's individual definition. The order of precedence for configurations (from highest to least precedence): + +- Compute resource individual configuration + - Compute resource family default configuration (`eks_managed_node_group_defaults`, `self_managed_node_group_defaults`, `fargate_profile_defaults`) + - Module default configuration (see `variables.tf` and `node_groups.tf`) + +For example, the following creates 4 AWS EKS Managed Node Groups: + +```hcl + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] + } + + eks_managed_node_groups = { + # Uses module default configurations overridden by configuration above + default = {} + + # This further overrides the instance types used + compute = { + instance_types = ["c5.large", "c6i.large", "c6d.large"] + } + + # This further overrides the instance types and disk size used + persistent = { + disk_size = 1024 + instance_types = ["r5.xlarge", "r6i.xlarge", "r5b.xlarge"] + } + + # This overrides the OS used + bottlerocket = { + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + } + } +``` diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000000..d805a5b149 --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,110 @@ +# Frequently Asked Questions + +- [How do I manage the `aws-auth` configmap?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-do-i-manage-the-aws-auth-configmap) +- [I received an error: `Error: Invalid for_each argument ...`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#i-received-an-error-error-invalid-for_each-argument-) +- [Why are nodes not being registered?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-nodes-not-being-registered) +- [Why are there no changes when a node group's `desired_size` is modified?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-there-no-changes-when-a-node-groups-desired_size-is-modified) +- [How can I deploy Windows based nodes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-can-i-deploy-windows-based-nodes) +- [How do I access compute resource attributes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-do-i-access-compute-resource-attributes) + +### How do I manage the `aws-auth` configmap? + +TL;DR - https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1901 + +- Users can roll their own equivalent of `kubectl patch ...` using the [`null_resource`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/9a99689cc13147f4afc426b34ba009875a28614e/examples/complete/main.tf#L301-L336) +- There is a module that was created to fill this gap that provides a Kubernetes based approach to provision: https://github.com/aidanmelen/terraform-aws-eks-auth +- Ideally, one of the following issues are resolved upstream for a more native experience for users: + - https://github.com/aws/containers-roadmap/issues/185 + - https://github.com/hashicorp/terraform-provider-kubernetes/issues/723 + +### I received an error: `Error: Invalid for_each argument ...` + +Users may encounter an error such as `Error: Invalid for_each argument - The "for_each" value depends on resource attributes that cannot be determined until apply, so Terraform cannot predict how many instances will be created. To work around this, use the -target argument to first apply ...` + +This error is due to an upstream issue with [Terraform core](https://github.com/hashicorp/terraform/issues/4149). There are two potential options you can take to help mitigate this issue: + +1. Create the dependent resources before the cluster => `terraform apply -target ` and then `terraform apply` for the cluster (or other similar means to just ensure the referenced resources exist before creating the cluster) + +- Note: this is the route users will have to take for adding additional security groups to nodes since there isn't a separate "security group attachment" resource + +2. For additional IAM policies, users can attach the policies outside of the cluster definition as demonstrated below + +```hcl +resource "aws_iam_role_policy_attachment" "additional" { + for_each = module.eks.eks_managed_node_groups + # you could also do the following or any combination: + # for_each = merge( + # module.eks.eks_managed_node_groups, + # module.eks.self_managed_node_group, + # module.eks.fargate_profile, + # ) + + # This policy does not have to exist at the time of cluster creation. Terraform can + # deduce the proper order of its creation to avoid errors during creation + policy_arn = aws_iam_policy.node_additional.arn + role = each.value.iam_role_name +} +``` + +TL;DR - Terraform resource passed into the modules map definition _must_ be known before you can apply the EKS module. The variables this potentially affects are: + +- `cluster_security_group_additional_rules` (i.e. - referencing an external security group resource in a rule) +- `node_security_group_additional_rules` (i.e. - referencing an external security group resource in a rule) +- `iam_role_additional_policies` (i.e. - referencing an external policy resource) + +- Setting `instance_refresh_enabled = true` will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. See the [instance_refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa_autoscale_refresh) example provided. + +### Why are nodes not being registered? + +Nodes not being able to register with the EKS control plane is generally due to networking mis-configurations. + +1. At least one of the cluster endpoints (public or private) must be enabled. + +If you require a public endpoint, setting up both (public and private) and restricting the public endpoint via setting `cluster_endpoint_public_access_cidrs` is recommended. More info regarding communication with an endpoint is available [here](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html). + +2. Nodes need to be able to contact the EKS cluster endpoint. By default, the module only creates a public endpoint. To access the endpoint, the nodes need outgoing internet access: + +- Nodes in private subnets: via a NAT gateway or instance along with the appropriate routing rules +- Nodes in public subnets: ensure that nodes are launched with public IPs (enable through either the module here or your subnet setting defaults) + +**Important: If you apply only the public endpoint and configure the `cluster_endpoint_public_access_cidrs` to restrict access, know that EKS nodes will also use the public endpoint and you must allow access to the endpoint. If not, then your nodes will fail to work correctly.** + +3. The private endpoint can also be enabled by setting `cluster_endpoint_private_access = true`. Ensure that VPC DNS resolution and hostnames are also enabled for your VPC when the private endpoint is enabled. + +4. Nodes need to be able to connect to other AWS services to function (download container images, make API calls to assume roles, etc.). If for some reason you cannot enable public internet access for nodes you can add VPC endpoints to the relevant services: EC2 API, ECR API, ECR DKR and S3. + +### Why are there no changes when a node group's `desired_size` is modified? + +The module is configured to ignore this value. Unfortunately, Terraform does not support variables within the `lifecycle` block. The setting is ignored to allow autoscaling via controllers such as cluster autoscaler or Karpenter to work properly and without interference by Terraform. Changing the desired count must be handled outside of Terraform once the node group is created. + +### How can I deploy Windows based nodes? + +To enable Windows support for your EKS cluster, you will need to apply some configuration manually. See the [Enabling Windows Support (Windows/MacOS/Linux)](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support). + +In addition, Windows based nodes require an additional cluster RBAC role (`eks:kube-proxy-windows`). + +Note: Windows based node support is limited to a default user data template that is provided due to the lack of Windows support and manual steps required to provision Windows based EKS nodes. + +### How do I access compute resource attributes? + +Examples of accessing the attributes of the compute resource(s) created by the root module are shown below. Note - the assumption is that your cluster module definition is named `eks` as in `module "eks" { ... }`: + +````hcl + +- EKS Managed Node Group attributes + +```hcl +eks_managed_role_arns = [for group in module.eks_managed_node_group : group.iam_role_arn] +```` + +- Self Managed Node Group attributes + +```hcl +self_managed_role_arns = [for group in module.self_managed_node_group : group.iam_role_arn] +``` + +- Fargate Profile attributes + +```hcl +fargate_profile_pod_execution_role_arns = [for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn] +``` diff --git a/docs/irsa_integration.md b/docs/irsa_integration.md new file mode 100644 index 0000000000..93293e7705 --- /dev/null +++ b/docs/irsa_integration.md @@ -0,0 +1,84 @@ + +### IRSA Integration + +An [IAM role for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) module has been created to work in conjunction with this module. The [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) module has a set of pre-defined IAM policies for common addons. Check [`policy.tf`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/modules/iam-role-for-service-accounts-eks/policies.tf) for a list of the policies currently supported. One example of this integration is shown below, and more can be found in the [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/examples/iam-role-for-service-accounts-eks/main.tf) example directory: + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + + cluster_name = "example" + cluster_version = "1.21" + + cluster_addons = { + vpc-cni = { + resolve_conflicts = "OVERWRITE" + service_account_role_arn = module.vpc_cni_irsa.iam_role_arn + } + } + + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + + eks_managed_node_group_defaults = { + # We are using the IRSA created below for permissions + # However, we have to provision a new cluster with the policy attached FIRST + # before we can disable. Without this initial policy, + # the VPC CNI fails to assign IPs and nodes cannot join the new cluster + iam_role_attach_cni_policy = true + } + + eks_managed_node_groups = { + default = {} + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} + +module "vpc_cni_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + + role_name = "vpc_cni" + attach_vpc_cni_policy = true + vpc_cni_enable_ipv4 = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-node"] + } + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} + +module "karpenter_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + + role_name = "karpenter_controller" + attach_karpenter_controller_policy = true + + karpenter_controller_cluster_id = module.eks.cluster_id + karpenter_controller_node_iam_role_arns = [ + module.eks.eks_managed_node_groups["default"].iam_role_arn + ] + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["karpenter:karpenter"] + } + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` diff --git a/docs/network_connectivity.md b/docs/network_connectivity.md new file mode 100644 index 0000000000..67805aa77c --- /dev/null +++ b/docs/network_connectivity.md @@ -0,0 +1,68 @@ +# Network Connectivity + +## Cluster Endpoint + +### Public Endpoint w/ Restricted CIDRs + +When restricting the clusters public endpoint to only the CIDRs specified by users, it is recommended that you also enable the private endpoint, or ensure that the CIDR blocks that you specify include the addresses that nodes and Fargate pods (if you use them) access the public endpoint from. + +Please refer to the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html) for further information + +## Security Groups + +- Cluster Security Group + - This module by default creates a cluster security group ("additional" security group when viewed from the console) in addition to the default security group created by the AWS EKS service. This "additional" security group allows users to customize inbound and outbound rules via the module as they see fit + - The default inbound/outbound rules provided by the module are derived from the [AWS minimum recommendations](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) in addition to NTP and HTTPS public internet egress rules (without, these show up in VPC flow logs as rejects - they are used for clock sync and downloading necessary packages/updates) + - The minimum inbound/outbound rules are provided for cluster and node creation to succeed without errors, but users will most likely need to add the necessary port and protocol for node-to-node communication (this is user specific based on how nodes are configured to communicate across the cluster) + - Users have the ability to opt out of the security group creation and instead provide their own externally created security group if so desired + - The security group that is created is designed to handle the bare minimum communication necessary between the control plane and the nodes, as well as any external egress to allow the cluster to successfully launch without error + - Users also have the option to supply additional, externally created security groups to the cluster as well via the `cluster_additional_security_group_ids` variable + - Lastly, users are able to opt in to attaching the primary security group automatically created by the EKS service by setting `attach_cluster_primary_security_group` = `true` from the root module for the respective node group (or set it within the node group defaults). This security group is not managed by the module; it is created by the EKS service. It permits all traffic within the domain of the security group as well as all egress traffic to the internet. + +- Node Group Security Group(s) + - Each node group (EKS Managed Node Group and Self Managed Node Group) by default creates its own security group. By default, this security group does not contain any additional security group rules. It is merely an "empty container" that offers users the ability to opt into any addition inbound our outbound rules as necessary + - Users also have the option to supply their own, and/or additional, externally created security group(s) to the node group as well via the `vpc_security_group_ids` variable + +See the example snippet below which adds additional security group rules to the cluster security group as well as the shared node security group (for node-to-node access). Users can use this extensibility to open up network access as they see fit using the security groups provided by the module: + +```hcl + ... + # Extend cluster security group rules + cluster_security_group_additional_rules = { + egress_nodes_ephemeral_ports_tcp = { + description = "To node 1025-65535" + protocol = "tcp" + from_port = 1025 + to_port = 65535 + type = "egress" + source_node_security_group = true + } + } + + # Extend node-to-node security group rules + node_security_group_additional_rules = { + ingress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + egress_all = { + description = "Node all egress" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + } + ... +``` +The security groups created by this module are depicted in the image shown below along with their default inbound/outbound rules: + +

+ Security Groups +

diff --git a/docs/user_data.md b/docs/user_data.md new file mode 100644 index 0000000000..e5c247b798 --- /dev/null +++ b/docs/user_data.md @@ -0,0 +1,97 @@ +# User Data & Bootstrapping + +Users can see the various methods of using and providing user data through the [user data examples](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) as well more detailed information on the design and possible configurations via the [user data module itself](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/_user_data) + +## Summary + +- AWS EKS Managed Node Groups + - By default, any supplied user data is pre-pended to the user data supplied by the EKS Managed Node Group service + - If users supply an `ami_id`, the service no longers supplies user data to bootstrap nodes; users can enable `enable_bootstrap_user_data` and use the module provided user data template, or provide their own user data template + - `bottlerocket` platform user data must be in TOML format +- Self Managed Node Groups + - `linux` platform (default) -> the user data template (bash/shell script) provided by the module is used as the default; users are able to provide their own user data template + - `bottlerocket` platform -> the user data template (TOML file) provided by the module is used as the default; users are able to provide their own user data template + - `windows` platform -> the user data template (powershell/PS1 script) provided by the module is used as the default; users are able to provide their own user data template + +The templates provided by the module can be found under the [templates directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/templates) + +## EKS Managed Node Group + +When using an EKS managed node group, users have 2 primary routes for interacting with the bootstrap user data: + +1. If a value for `ami_id` is not provided, users can supply additional user data that is pre-pended before the EKS Managed Node Group bootstrap user data. You can read more about this process from the [AWS supplied documentation](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data) + + - Users can use the following variables to facilitate this process: + + ```hcl + pre_bootstrap_user_data = "..." + ``` + +2. If a custom AMI is used, then per the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami), users will need to supply the necessary user data to bootstrap and register nodes with the cluster when launched. There are two routes to facilitate this bootstrapping process: + - If the AMI used is a derivative of the [AWS EKS Optimized AMI ](https://github.com/awslabs/amazon-eks-ami), users can opt in to using a template provided by the module that provides the minimum necessary configuration to bootstrap the node when launched: + - Users can use the following variables to facilitate this process: + ```hcl + enable_bootstrap_user_data = true # to opt in to using the module supplied bootstrap user data template + pre_bootstrap_user_data = "..." + bootstrap_extra_args = "..." + post_bootstrap_user_data = "..." + ``` + - If the AMI is **NOT** an AWS EKS Optimized AMI derivative, or if users wish to have more control over the user data that is supplied to the node when launched, users have the ability to supply their own user data template that will be rendered instead of the module supplied template. Note - only the variables that are supplied to the `templatefile()` for the respective platform/OS are available for use in the supplied template, otherwise users will need to pre-render/pre-populate the template before supplying the final template to the module for rendering as user data. + - Users can use the following variables to facilitate this process: + ```hcl + user_data_template_path = "./your/user_data.sh" # user supplied bootstrap user data template + pre_bootstrap_user_data = "..." + bootstrap_extra_args = "..." + post_bootstrap_user_data = "..." + ``` + +| ℹ️ When using bottlerocket as the desired platform, since the user data for bottlerocket is TOML, all configurations are merged in the one file supplied as user data. Therefore, `pre_bootstrap_user_data` and `post_bootstrap_user_data` are not valid since the bottlerocket OS handles when various settings are applied. If you wish to supply additional configuration settings when using bottlerocket, supply them via the `bootstrap_extra_args` variable. For the linux platform, `bootstrap_extra_args` are settings that will be supplied to the [AWS EKS Optimized AMI bootstrap script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh#L14) such as kubelet extra args, etc. See the [bottlerocket GitHub repository documentation](https://github.com/bottlerocket-os/bottlerocket#description-of-settings) for more details on what settings can be supplied via the `bootstrap_extra_args` variable. | +| :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | + +#### ⚠️ Caveat + +Since the EKS Managed Node Group service provides the necessary bootstrap user data to nodes (unless an `ami_id` is provided), users do not have direct access to settings/variables provided by the EKS optimized AMI [`bootstrap.sh` script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh). Currently, users must employ work-arounds to influence the `bootstrap.sh` script. For example, to enable `containerd` on EKS Managed Node Groups, users can supply the following user data. You can learn more about this issue [here](https://github.com/awslabs/amazon-eks-ami/issues/844): + +```hcl + # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 + pre_bootstrap_user_data = <<-EOT + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + export KUBELET_EXTRA_ARGS="--max-pods=110" + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + EOT +``` + +### Self Managed Node Group + +Self managed node groups require users to provide the necessary bootstrap user data. Users can elect to use the user data template provided by the module for their platform/OS or provide their own user data template for rendering by the module. + +- If the AMI used is a derivative of the [AWS EKS Optimized AMI ](https://github.com/awslabs/amazon-eks-ami), users can opt in to using a template provided by the module that provides the minimum necessary configuration to bootstrap the node when launched: + - Users can use the following variables to facilitate this process: + ```hcl + enable_bootstrap_user_data = true # to opt in to using the module supplied bootstrap user data template + pre_bootstrap_user_data = "..." + bootstrap_extra_args = "..." + post_bootstrap_user_data = "..." + ``` + - If the AMI is **NOT** an AWS EKS Optimized AMI derivative, or if users wish to have more control over the user data that is supplied to the node when launched, users have the ability to supply their own user data template that will be rendered instead of the module supplied template. Note - only the variables that are supplied to the `templatefile()` for the respective platform/OS are available for use in the supplied template, otherwise users will need to pre-render/pre-populate the template before supplying the final template to the module for rendering as user data. + - Users can use the following variables to facilitate this process: + ```hcl + user_data_template_path = "./your/user_data.sh" # user supplied bootstrap user data template + pre_bootstrap_user_data = "..." + bootstrap_extra_args = "..." + post_bootstrap_user_data = "..." + ``` + +### Logic Diagram + +The rough flow of logic that is encapsulated within the `_user_data` module can be represented by the following diagram to better highlight the various manners in which user data can be populated. + +

+ User Data +

diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000000..f417c0adc0 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,8 @@ +# Examples + +Please note - the examples provided serve two primary means: + +1. Show users working examples of the various ways in which the module can be configured and features supported +2. A means of testing/validating module changes + +Please do not mistake the examples provided as "best practices". It is up to users to consult the AWS service documentation for best practices, usage recommendations, etc. diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 51e7b9292e..87da77b447 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -1,77 +1,8 @@ -# Internal User Data Module +# User Data Module -Configuration in this directory renders the appropriate user data for the given inputs. There are a number of different ways that user data can be utilized and this internal module is designed to aid in making that flexibility possible as well as providing a means for out of bands testing and validation. +Configuration in this directory renders the appropriate user data for the given inputs. See [`docs/user_data.md`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/user_data.md) for more info. -See the [`examples/user_data/` directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) for various examples of using the module. - -## Combinations - -At a high level, AWS EKS users have two methods for launching nodes within this EKS module (ignoring Fargate profiles): - -1. EKS managed node group -2. Self managed node group - -### EKS Managed Node Group - -When using an EKS managed node group, users have 2 primary routes for interacting with the bootstrap user data: - -1. If the EKS managed node group does **NOT** utilize a custom AMI, then users can elect to supply additional user data that is pre-pended before the EKS managed node group bootstrap user data. You can read more about this process from the [AWS supplied documentation](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data) - - - Users can use the following variables to facilitate this process: - - ```hcl - pre_bootstrap_user_data = "..." - ``` - -2. If the EKS managed node group does utilize a custom AMI, then per the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami), users will need to supply the necessary bootstrap configuration via user data to ensure that the node is configured to register with the cluster when launched. There are two routes that users can utilize to facilitate this bootstrapping process: - - If the AMI used is a derivative of the [AWS EKS Optimized AMI ](https://github.com/awslabs/amazon-eks-ami), users can opt in to using a template provided by the module that provides the minimum necessary configuration to bootstrap the node when launched, with the option to add additional pre and post bootstrap user data as well as bootstrap additional args that are supplied to the [AWS EKS bootstrap.sh script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh) - - Users can use the following variables to facilitate this process: - ```hcl - enable_bootstrap_user_data = true # to opt in to using the module supplied bootstrap user data template - pre_bootstrap_user_data = "..." - bootstrap_extra_args = "..." - post_bootstrap_user_data = "..." - ``` - - If the AMI is not an AWS EKS Optimized AMI derivative, or if users wish to have more control over the user data that is supplied to the node when launched, users have the ability to supply their own user data template that will be rendered instead of the module supplied template. Note - only the variables that are supplied to the `templatefile()` for the respective platform/OS are available for use in the supplied template, otherwise users will need to pre-render/pre-populate the template before supplying the final template to the module for rendering as user data. - - Users can use the following variables to facilitate this process: - ```hcl - user_data_template_path = "./your/user_data.sh" # user supplied bootstrap user data template - pre_bootstrap_user_data = "..." - bootstrap_extra_args = "..." - post_bootstrap_user_data = "..." - ``` - -| ℹ️ When using bottlerocket as the desired platform, since the user data for bottlerocket is TOML, all configurations are merged in the one file supplied as user data. Therefore, `pre_bootstrap_user_data` and `post_bootstrap_user_data` are not valid since the bottlerocket OS handles when various settings are applied. If you wish to supply additional configuration settings when using bottlerocket, supply them via the `bootstrap_extra_args` variable. For the linux platform, `bootstrap_extra_args` are settings that will be supplied to the [AWS EKS Optimized AMI bootstrap script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh#L14) such as kubelet extra args, etc. See the [bottlerocket GitHub repository documentation](https://github.com/bottlerocket-os/bottlerocket#description-of-settings) for more details on what settings can be supplied via the `bootstrap_extra_args` variable. | -| :--- | - -### Self Managed Node Group - -When using a self managed node group, the options presented to users is very similar to the 2nd option listed above for EKS managed node groups. Since self managed node groups require users to provide the bootstrap user data, there is no concept of appending to user data that AWS provides; users can either elect to use the user data template provided for their platform/OS by the module or provide their own user data template for rendering by the module. - -- If the AMI used is a derivative of the [AWS EKS Optimized AMI ](https://github.com/awslabs/amazon-eks-ami), users can opt in to using a template provided by the module that provides the minimum necessary configuration to bootstrap the node when launched, with the option to add additional pre and post bootstrap user data as well as bootstrap additional args that are supplied to the [AWS EKS bootstrap.sh script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh) - - Users can use the following variables to facilitate this process: - ```hcl - enable_bootstrap_user_data = true # to opt in to using the module supplied bootstrap user data template - pre_bootstrap_user_data = "..." - bootstrap_extra_args = "..." - post_bootstrap_user_data = "..." - ``` -- If the AMI is not an AWS EKS Optimized AMI derivative, or if users wish to have more control over the user data that is supplied to the node upon launch, users have the ability to supply their own user data template that will be rendered instead of the module supplied template. Note - only the variables that are supplied to the `templatefile()` for the respective platform/OS are available for use in the supplied template, otherwise users will need to pre-render/pre-populate the template before supplying the final template to the module for rendering as user data. - - Users can use the following variables to facilitate this process: - ```hcl - user_data_template_path = "./your/user_data.sh" # user supplied bootstrap user data template - pre_bootstrap_user_data = "..." - bootstrap_extra_args = "..." - post_bootstrap_user_data = "..." - ``` - -### Logic Diagram - -The rough flow of logic that is encapsulated within the `_user_data` internal module can be represented by the following diagram to better highlight the various manners in which user data can be populated. - -

- User Data -

+See [`examples/user_data/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) for various examples using this module. ## Requirements From 39a7c0c1f0b077d1d1bec1ab955a5249f752d27e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 3 Apr 2022 14:52:49 -0400 Subject: [PATCH 525/962] docs: Fix broken link and correct doc mis-spelling (#1983) --- README.md | 2 +- docs/README.md | 2 +- docs/UPGRADE-18.0.md | 2 +- docs/{compute_resourcs.md => compute_resources.md} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename docs/{compute_resourcs.md => compute_resources.md} (100%) diff --git a/README.md b/README.md index e1fd75405e..8d51747425 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Terraform module which creates AWS EKS (Kubernetes) resources - [Frequently Asked Questions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md) - [Compute Resources](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/compute_resources.md) -- [IRSA Integration](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/irsa-integration.md) +- [IRSA Integration](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/irsa_integration.md) - [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/user_data.md) - [Network Connectivity](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/network_connectivity.md) - Upgrade Guides diff --git a/docs/README.md b/docs/README.md index ccbb3f8ee7..3806b326e5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,7 +4,7 @@ - [Frequently Asked Questions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md) - [Compute Resources](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/compute_resources.md) -- [IRSA Integration](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/irsa-integration.md) +- [IRSA Integration](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/irsa_integration.md) - [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/user_data.md) - [Network Connectivity](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/network_connectivity.md) - Upgrade Guides diff --git a/docs/UPGRADE-18.0.md b/docs/UPGRADE-18.0.md index 29ae3ebcd7..33c1767be5 100644 --- a/docs/UPGRADE-18.0.md +++ b/docs/UPGRADE-18.0.md @@ -2,7 +2,7 @@ Please consult the `examples` directory for reference example configurations. If you find a bug, please open an issue with supporting configuration to reproduce. -Note: please see https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1744 where users have shared their steps/information for their individual configurations. Due to the numerous configuration possibilities, it is difficult to capture specific steps that will work for all and this has been a very helpful issue for others to share they were able to upgrade. +Note: please see https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1744 where users have shared the steps/changes that have worked for their configurations to upgrade. Due to the numerous configuration possibilities, it is difficult to capture specific steps that will work for all; this has proven to be a useful thread to share collective information from the broader community regarding v18.x upgrades. ## List of backwards incompatible changes diff --git a/docs/compute_resourcs.md b/docs/compute_resources.md similarity index 100% rename from docs/compute_resourcs.md rename to docs/compute_resources.md From 6db89f8f20a58ae5cfbab5541ff7e499ddf971b8 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 4 Apr 2022 10:34:29 -0400 Subject: [PATCH 526/962] feat: Add `create_before_destroy` lifecycle hook to security groups created (#1985) --- main.tf | 4 ++++ modules/eks-managed-node-group/main.tf | 6 ++++++ modules/self-managed-node-group/main.tf | 6 ++++++ node_groups.tf | 4 ++++ 4 files changed, 20 insertions(+) diff --git a/main.tf b/main.tf index 31568d28ac..0e802d82f8 100644 --- a/main.tf +++ b/main.tf @@ -129,6 +129,10 @@ resource "aws_security_group" "cluster" { { "Name" = local.cluster_sg_name }, var.cluster_security_group_tags ) + + lifecycle { + create_before_destroy = true + } } resource "aws_security_group_rule" "cluster" { diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 774fcef378..b18335de20 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -372,6 +372,12 @@ resource "aws_security_group" "this" { { "Name" = local.security_group_name }, var.security_group_tags ) + + # https://github.com/hashicorp/terraform-provider-aws/issues/2445 + # https://github.com/hashicorp/terraform-provider-aws/issues/9692 + lifecycle { + create_before_destroy = true + } } resource "aws_security_group_rule" "this" { diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 6278cecf38..cb3c70db85 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -458,6 +458,12 @@ resource "aws_security_group" "this" { }, var.security_group_tags ) + + # https://github.com/hashicorp/terraform-provider-aws/issues/2445 + # https://github.com/hashicorp/terraform-provider-aws/issues/9692 + lifecycle { + create_before_destroy = true + } } resource "aws_security_group_rule" "this" { diff --git a/node_groups.tf b/node_groups.tf index 4d8e0d787a..09c3747e1d 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -161,6 +161,10 @@ resource "aws_security_group" "node" { }, var.node_security_group_tags ) + + lifecycle { + create_before_destroy = true + } } resource "aws_security_group_rule" "node" { From 20af82846b4a1f23f3787a8c455f39c0b6164d80 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 4 Apr 2022 14:35:14 +0000 Subject: [PATCH 527/962] chore(release): version 18.19.0 [skip ci] ## [18.19.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.18.0...v18.19.0) (2022-04-04) ### Features * Add `create_before_destroy` lifecycle hook to security groups created ([#1985](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1985)) ([6db89f8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6db89f8f20a58ae5cfbab5541ff7e499ddf971b8)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 957083ebca..602f9074df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.19.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.18.0...v18.19.0) (2022-04-04) + + +### Features + +* Add `create_before_destroy` lifecycle hook to security groups created ([#1985](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1985)) ([6db89f8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6db89f8f20a58ae5cfbab5541ff7e499ddf971b8)) + ## [18.18.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.17.1...v18.18.0) (2022-04-03) From 3ff17205a4ead51cca993547ef3de42cc080043b Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 7 Apr 2022 14:47:22 -0400 Subject: [PATCH 528/962] docs: Replace cluster autoscaler/node termination handler example with Karpenter (#1994) --- .pre-commit-config.yaml | 2 +- README.md | 2 +- docs/faq.md | 2 - examples/eks_managed_node_group/main.tf | 2 +- examples/fargate_profile/main.tf | 2 +- examples/irsa_autoscale_refresh/charts.tf | 227 ---------------- examples/irsa_autoscale_refresh/main.tf | 188 ------------- .../README.md | 68 +++-- examples/karpenter/main.tf | 248 ++++++++++++++++++ .../outputs.tf | 0 .../variables.tf | 0 .../versions.tf | 10 +- examples/self_managed_node_group/main.tf | 2 +- 13 files changed, 301 insertions(+), 452 deletions(-) delete mode 100644 examples/irsa_autoscale_refresh/charts.tf delete mode 100644 examples/irsa_autoscale_refresh/main.tf rename examples/{irsa_autoscale_refresh => karpenter}/README.md (63%) create mode 100644 examples/karpenter/main.tf rename examples/{irsa_autoscale_refresh => karpenter}/outputs.tf (100%) rename examples/{irsa_autoscale_refresh => karpenter}/variables.tf (100%) rename examples/{irsa_autoscale_refresh => karpenter}/versions.tf (66%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f00fbfe8aa..ba45548ddd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - '--args=--only=terraform_standard_module_structure' - '--args=--only=terraform_workspace_remote' - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.2.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer diff --git a/README.md b/README.md index 8d51747425..dfa75c6282 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ module "eks" { - [Complete](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete): EKS Cluster using all available node group types in various combinations demonstrating many of the supported features and configurations - [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group): EKS Cluster using EKS managed node groups - [Fargate Profile](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile): EKS cluster using [Fargate Profiles](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) -- [IRSA, Node Autoscaler, Instance Refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa_autoscale_refresh): EKS Cluster using self-managed node group demonstrating how to enable/utilize instance refresh configuration along with node termination handler +- [Karpenter](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/karpenter): EKS Cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling - [Self Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self_managed_node_group): EKS Cluster using self-managed node groups - [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data): Various supported methods of providing necessary bootstrap scripts and configuration settings via user data diff --git a/docs/faq.md b/docs/faq.md index d805a5b149..33e8ad7964 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -52,8 +52,6 @@ TL;DR - Terraform resource passed into the modules map definition _must_ be know - `node_security_group_additional_rules` (i.e. - referencing an external security group resource in a rule) - `iam_role_additional_policies` (i.e. - referencing an external policy resource) -- Setting `instance_refresh_enabled = true` will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. See the [instance_refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa_autoscale_refresh) example provided. - ### Why are nodes not being registered? Nodes not being able to register with the EKS control plane is generally due to networking mis-configurations. diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 225c5140d0..8fc40db12f 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -4,7 +4,7 @@ provider "aws" { locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.21" + cluster_version = "1.22" region = "eu-west-1" tags = { diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index 6106a272b4..6c80e22c41 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -4,7 +4,7 @@ provider "aws" { locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.21" + cluster_version = "1.22" region = "eu-west-1" tags = { diff --git a/examples/irsa_autoscale_refresh/charts.tf b/examples/irsa_autoscale_refresh/charts.tf deleted file mode 100644 index 956b8de9f4..0000000000 --- a/examples/irsa_autoscale_refresh/charts.tf +++ /dev/null @@ -1,227 +0,0 @@ -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - token = data.aws_eks_cluster_auth.cluster.token - } -} - -################################################################################ -# Cluster Autoscaler -# Based on the official docs at -# https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler -################################################################################ - -resource "helm_release" "cluster_autoscaler" { - name = "cluster-autoscaler" - namespace = "kube-system" - repository = "https://kubernetes.github.io/autoscaler" - chart = "cluster-autoscaler" - version = "9.10.8" - create_namespace = false - - set { - name = "awsRegion" - value = local.region - } - - set { - name = "rbac.serviceAccount.name" - value = "cluster-autoscaler-aws" - } - - set { - name = "rbac.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.cluster_autoscaler_irsa.iam_role_arn - type = "string" - } - - set { - name = "autoDiscovery.clusterName" - value = local.name - } - - set { - name = "autoDiscovery.enabled" - value = "true" - } - - set { - name = "rbac.create" - value = "true" - } - - depends_on = [ - module.eks.cluster_id, - null_resource.apply, - ] -} - -module "cluster_autoscaler_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 4.12" - - role_name_prefix = "cluster-autoscaler" - role_description = "IRSA role for cluster autoscaler" - - attach_cluster_autoscaler_policy = true - cluster_autoscaler_cluster_ids = [module.eks.cluster_id] - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:cluster-autoscaler-aws"] - } - } - - tags = local.tags -} - -################################################################################ -# Node Termination Handler -# Based on the official docs at -# https://github.com/aws/aws-node-termination-handler -################################################################################ - -resource "helm_release" "aws_node_termination_handler" { - name = "aws-node-termination-handler" - namespace = "kube-system" - repository = "https://aws.github.io/eks-charts" - chart = "aws-node-termination-handler" - version = "0.16.0" - create_namespace = false - - set { - name = "awsRegion" - value = local.region - } - - set { - name = "serviceAccount.name" - value = "aws-node-termination-handler" - } - - set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.node_termination_handler_irsa.iam_role_arn - type = "string" - } - - set { - name = "enableSqsTerminationDraining" - value = "true" - } - - set { - name = "enableSpotInterruptionDraining" - value = "true" - } - - set { - name = "queueURL" - value = module.aws_node_termination_handler_sqs.sqs_queue_id - } - - set { - name = "logLevel" - value = "debug" - } - - depends_on = [ - module.eks.cluster_id, - null_resource.apply, - ] -} - -module "node_termination_handler_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 4.12" - - role_name_prefix = "node-termination-handler" - role_description = "IRSA role for node termination handler" - - attach_node_termination_handler_policy = true - node_termination_handler_sqs_queue_arns = [module.aws_node_termination_handler_sqs.sqs_queue_arn] - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:aws-node-termination-handler"] - } - } - - tags = local.tags -} - -module "aws_node_termination_handler_sqs" { - source = "terraform-aws-modules/sqs/aws" - version = "~> 3.0" - - name = local.name - message_retention_seconds = 300 - policy = data.aws_iam_policy_document.aws_node_termination_handler_sqs.json - - tags = local.tags -} - -data "aws_iam_policy_document" "aws_node_termination_handler_sqs" { - statement { - actions = ["sqs:SendMessage"] - resources = ["arn:aws:sqs:${local.region}:${data.aws_caller_identity.current.account_id}:${local.name}"] - - principals { - type = "Service" - identifiers = [ - "events.amazonaws.com", - "sqs.amazonaws.com", - ] - } - } -} - -resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_asg" { - name = "${local.name}-asg-termination" - description = "Node termination event rule" - - event_pattern = jsonencode({ - "source" : ["aws.autoscaling"], - "detail-type" : ["EC2 Instance-terminate Lifecycle Action"] - "resources" : [for group in module.eks.self_managed_node_groups : group.autoscaling_group_arn] - }) - - tags = local.tags -} - -resource "aws_cloudwatch_event_target" "aws_node_termination_handler_asg" { - target_id = "${local.name}-asg-termination" - rule = aws_cloudwatch_event_rule.aws_node_termination_handler_asg.name - arn = module.aws_node_termination_handler_sqs.sqs_queue_arn -} - -resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_spot" { - name = "${local.name}-spot-termination" - description = "Node termination event rule" - event_pattern = jsonencode({ - "source" : ["aws.ec2"], - "detail-type" : ["EC2 Spot Instance Interruption Warning"] - }) -} - -resource "aws_cloudwatch_event_target" "aws_node_termination_handler_spot" { - target_id = "${local.name}-spot-termination" - rule = aws_cloudwatch_event_rule.aws_node_termination_handler_spot.name - arn = module.aws_node_termination_handler_sqs.sqs_queue_arn -} - -# Creating the lifecycle-hook outside of the ASG resource's `initial_lifecycle_hook` -# ensures that node termination does not require the lifecycle action to be completed, -# and thus allows the ASG to be destroyed cleanly. -resource "aws_autoscaling_lifecycle_hook" "aws_node_termination_handler" { - for_each = module.eks.self_managed_node_groups - - name = "aws-node-termination-handler-${each.value.autoscaling_group_name}" - autoscaling_group_name = each.value.autoscaling_group_name - lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" - heartbeat_timeout = 300 - default_result = "CONTINUE" -} diff --git a/examples/irsa_autoscale_refresh/main.tf b/examples/irsa_autoscale_refresh/main.tf deleted file mode 100644 index faf16dc208..0000000000 --- a/examples/irsa_autoscale_refresh/main.tf +++ /dev/null @@ -1,188 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.21" - region = "eu-west-1" - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -data "aws_caller_identity" "current" {} - -data "aws_eks_cluster_auth" "cluster" { - name = module.eks.cluster_id -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - # Self Managed Node Group(s) - self_managed_node_groups = { - refresh = { - max_size = 5 - desired_size = 1 - - instance_type = "m5.large" - - instance_refresh = { - strategy = "Rolling" - preferences = { - checkpoint_delay = 600 - checkpoint_percentages = [35, 70, 100] - instance_warmup = 300 - min_healthy_percentage = 50 - } - triggers = ["tag"] - } - - tags = { "aws-node-termination-handler/managed" = "true" } - } - - mixed_instance = { - use_mixed_instances_policy = true - mixed_instances_policy = { - instances_distribution = { - on_demand_base_capacity = 0 - on_demand_percentage_above_base_capacity = 10 - spot_allocation_strategy = "capacity-optimized" - } - - override = [ - { - instance_type = "m5.large" - weighted_capacity = "1" - }, - { - instance_type = "m6i.large" - weighted_capacity = "2" - }, - ] - } - - tags = { "aws-node-termination-handler/managed" = "true" } - } - - spot = { - instance_type = "m5.large" - instance_market_options = { - market_type = "spot" - } - - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" - tags = { "aws-node-termination-handler/managed" = "true" } - } - } - - tags = merge(local.tags, { Foo = "bar" }) -} - -################################################################################ -# aws-auth configmap -# Only EKS managed node groups automatically add roles to aws-auth configmap -# so we need to ensure fargate profiles and self-managed node roles are added -################################################################################ - -data "aws_eks_cluster_auth" "this" { - name = module.eks.cluster_id -} - -locals { - kubeconfig = yamlencode({ - apiVersion = "v1" - kind = "Config" - current-context = "terraform" - clusters = [{ - name = module.eks.cluster_id - cluster = { - certificate-authority-data = module.eks.cluster_certificate_authority_data - server = module.eks.cluster_endpoint - } - }] - contexts = [{ - name = "terraform" - context = { - cluster = module.eks.cluster_id - user = "terraform" - } - }] - users = [{ - name = "terraform" - user = { - token = data.aws_eks_cluster_auth.this.token - } - }] - }) -} - -resource "null_resource" "apply" { - triggers = { - kubeconfig = base64encode(local.kubeconfig) - cmd_patch = <<-EOT - kubectl create configmap aws-auth -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) - kubectl patch configmap/aws-auth --patch "${module.eks.aws_auth_configmap_yaml}" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) - EOT - } - - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - environment = { - KUBECONFIG = self.triggers.kubeconfig - } - command = self.triggers.cmd_patch - } -} - -################################################################################ -# Supporting Resources -################################################################################ - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = "10.0.0.0/16" - - azs = ["${local.region}a", "${local.region}b", "${local.region}c"] - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - enable_flow_log = true - create_flow_log_cloudwatch_iam_role = true - create_flow_log_cloudwatch_log_group = true - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = 1 - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = 1 - } - - tags = local.tags -} diff --git a/examples/irsa_autoscale_refresh/README.md b/examples/karpenter/README.md similarity index 63% rename from examples/irsa_autoscale_refresh/README.md rename to examples/karpenter/README.md index a331d6973f..53e6bca5eb 100644 --- a/examples/irsa_autoscale_refresh/README.md +++ b/examples/karpenter/README.md @@ -1,10 +1,6 @@ -# IRSA, Cluster Autoscaler, and Instance Refresh example +# Karpenter Example -Configuration in this directory creates an AWS EKS cluster with: -- [IAM Roles for Service Accounts (IRSA)](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) enabled -- [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md) provisioned via a Helm Chart manifest -- [Instance Refresh](https://aws.amazon.com/blogs/compute/introducing-instance-refresh-for-ec2-auto-scaling/) feature for self managed node groups -- [Node Termination Handler](https://github.com/aws/aws-node-termination-handler) provisioned via a Helm Chart manifest +Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. ## Usage @@ -16,6 +12,38 @@ $ terraform plan $ terraform apply ``` +Once the cluster is up and running, you can check that Karpenter is functioning as intended with the following command: + +```bash +# First, make sure you have updated your local kubeconfig +aws eks --region eu-west-1 update-kubeconfig --name ex-karpenter + +# Second, scale the example deployment +kubectl scale deployment inflate --replicas 5 + +# You can watch Karpenter's controller logs with +kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller +``` + +You should see a new node named `karpenter.sh/provisioner-name/default` eventually come up in the console; this was provisioned by Karpenter in response to the scaled deployment above. + +### Tear Down & Clean-Up + +Because Karpenter manages the state of node resources outside of Terraform, Karpenter created resources will need to be de-provisioned first before removing the remaining resources with Terraform. + +1. Remove the example deployment created above and any nodes created by Karpenter + +```bash +kubectl delete deployment inflate +kubectl delete node -l karpenter.sh/provisioner-name=default +``` + +2. Remove the resources created by Terraform + +```bash +terraform destroy +``` + Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. @@ -25,43 +53,33 @@ Note that this example may create resources which cost money. Run `terraform des |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | -| [helm](#requirement\_helm) | >= 2.0 | -| [null](#requirement\_null) | >= 3.0 | +| [helm](#requirement\_helm) | >= 2.4 | +| [kubectl](#requirement\_kubectl) | >= 1.14 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | -| [helm](#provider\_helm) | >= 2.0 | -| [null](#provider\_null) | >= 3.0 | +| [helm](#provider\_helm) | >= 2.4 | +| [kubectl](#provider\_kubectl) | >= 1.14 | ## Modules | Name | Source | Version | |------|--------|---------| -| [aws\_node\_termination\_handler\_sqs](#module\_aws\_node\_termination\_handler\_sqs) | terraform-aws-modules/sqs/aws | ~> 3.0 | -| [cluster\_autoscaler\_irsa](#module\_cluster\_autoscaler\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.12 | | [eks](#module\_eks) | ../.. | n/a | -| [node\_termination\_handler\_irsa](#module\_node\_termination\_handler\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.12 | +| [karpenter\_irsa](#module\_karpenter\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.15 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources | Name | Type | |------|------| -| [aws_autoscaling_lifecycle_hook.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_lifecycle_hook) | resource | -| [aws_cloudwatch_event_rule.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | -| [aws_cloudwatch_event_rule.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | -| [aws_cloudwatch_event_target.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | -| [aws_cloudwatch_event_target.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | -| [helm_release.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [null_resource.apply](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | -| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | -| [aws_iam_policy_document.aws_node_termination_handler_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_instance_profile.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | ## Inputs diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf new file mode 100644 index 0000000000..fd98f22fbb --- /dev/null +++ b/examples/karpenter/main.tf @@ -0,0 +1,248 @@ +provider "aws" { + region = local.region +} + +locals { + name = "ex-${replace(basename(path.cwd), "_", "-")}" + cluster_version = "1.22" + region = "eu-west-1" + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + node_security_group_additional_rules = { + # Control plane invoke Karpenter webhook + ingress_karpenter_webhook_tcp = { + description = "Control plane invoke Karpenter webhook" + protocol = "tcp" + from_port = 8443 + to_port = 8443 + type = "ingress" + source_cluster_security_group = true + } + } + + eks_managed_node_groups = { + karpenter = { + instance_types = ["t3.medium"] + + min_size = 1 + max_size = 2 + desired_size = 1 + + iam_role_additional_policies = [ + # Required by Karpenter + "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + ] + } + } + + tags = merge(local.tags, { + # NOTE - if creating multiple security groups with this module, only tag the + # security group that Karpenter should utilize with the following tag + # (i.e. - at most, only one security group should have this tag in your account) + "karpenter.sh/discovery" = local.name + }) +} + +################################################################################ +# Karpenter +################################################################################ + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1alpha1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + } + } +} + +provider "kubectl" { + apply_retry_count = 5 + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + load_config_file = false + + exec { + api_version = "client.authentication.k8s.io/v1alpha1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + } +} + +module "karpenter_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 4.15" + + role_name = "karpenter-controller-${local.name}" + attach_karpenter_controller_policy = true + + karpenter_controller_cluster_id = module.eks.cluster_id + karpenter_controller_node_iam_role_arns = [ + module.eks.eks_managed_node_groups["karpenter"].iam_role_arn + ] + + oidc_providers = { + ex = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["karpenter:karpenter"] + } + } +} + +resource "aws_iam_instance_profile" "karpenter" { + name = "KarpenterNodeInstanceProfile-${local.name}" + role = module.eks.eks_managed_node_groups["karpenter"].iam_role_name +} + +resource "helm_release" "karpenter" { + namespace = "karpenter" + create_namespace = true + + name = "karpenter" + repository = "https://charts.karpenter.sh" + chart = "karpenter" + version = "0.8.1" + + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.karpenter_irsa.iam_role_arn + } + + set { + name = "clusterName" + value = module.eks.cluster_id + } + + set { + name = "clusterEndpoint" + value = module.eks.cluster_endpoint + } + + set { + name = "aws.defaultInstanceProfile" + value = aws_iam_instance_profile.karpenter.name + } +} + +# Workaround - https://github.com/hashicorp/terraform-provider-kubernetes/issues/1380#issuecomment-967022975 +resource "kubectl_manifest" "karpenter_provisioner" { + yaml_body = <<-YAML + apiVersion: karpenter.sh/v1alpha5 + kind: Provisioner + metadata: + name: default + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + limits: + resources: + cpu: 1000 + provider: + subnetSelector: + karpenter.sh/discovery: ${local.name} + securityGroupSelector: + karpenter.sh/discovery: ${local.name} + tags: + karpenter.sh/discovery: ${local.name} + ttlSecondsAfterEmpty: 30 + YAML + + depends_on = [ + helm_release.karpenter + ] +} + +# Example deployment using the [pause image](https://www.ianlewis.org/en/almighty-pause-container) +# and starts with zero replicas +resource "kubectl_manifest" "karpenter_example_deployment" { + yaml_body = <<-YAML + apiVersion: apps/v1 + kind: Deployment + metadata: + name: inflate + spec: + replicas: 0 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 + resources: + requests: + cpu: 1 + YAML + + depends_on = [ + helm_release.karpenter + ] +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = "10.0.0.0/16" + + azs = ["${local.region}a", "${local.region}b", "${local.region}c"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + # Tags subnets for Karpenter auto-discovery + "karpenter.sh/discovery" = local.name + } + + tags = local.tags +} diff --git a/examples/irsa_autoscale_refresh/outputs.tf b/examples/karpenter/outputs.tf similarity index 100% rename from examples/irsa_autoscale_refresh/outputs.tf rename to examples/karpenter/outputs.tf diff --git a/examples/irsa_autoscale_refresh/variables.tf b/examples/karpenter/variables.tf similarity index 100% rename from examples/irsa_autoscale_refresh/variables.tf rename to examples/karpenter/variables.tf diff --git a/examples/irsa_autoscale_refresh/versions.tf b/examples/karpenter/versions.tf similarity index 66% rename from examples/irsa_autoscale_refresh/versions.tf rename to examples/karpenter/versions.tf index 20ac24f966..fe18abab81 100644 --- a/examples/irsa_autoscale_refresh/versions.tf +++ b/examples/karpenter/versions.tf @@ -6,13 +6,13 @@ terraform { source = "hashicorp/aws" version = ">= 3.72" } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } helm = { source = "hashicorp/helm" - version = ">= 2.0" + version = ">= 2.4" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.14" } } } diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 373567b897..a7ad36cc81 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -4,7 +4,7 @@ provider "aws" { locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.21" + cluster_version = "1.22" region = "eu-west-1" tags = { From da3d54cde70adfd8b5d2770805b17d526923113e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 9 Apr 2022 03:15:46 -0400 Subject: [PATCH 529/962] feat: Add support for managing `aws-auth` configmap using new `kubernetes_config_map_v1_data` resource (#1999) --- README.md | 14 ++- examples/complete/README.md | 5 +- examples/complete/main.tf | 123 ++++++++----------- examples/complete/versions.tf | 6 +- examples/eks_managed_node_group/README.md | 5 +- examples/eks_managed_node_group/main.tf | 67 +++------- examples/eks_managed_node_group/versions.tf | 8 +- examples/fargate_profile/README.md | 1 + examples/fargate_profile/versions.tf | 4 + examples/self_managed_node_group/README.md | 5 +- examples/self_managed_node_group/main.tf | 72 +++-------- examples/self_managed_node_group/versions.tf | 8 +- main.tf | 95 ++++++++++++++ outputs.tf | 2 +- variables.tf | 52 ++++++++ versions.tf | 4 + 16 files changed, 263 insertions(+), 208 deletions(-) diff --git a/README.md b/README.md index dfa75c6282..a98eb30712 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | | [tls](#requirement\_tls) | >= 3.0 | ## Providers @@ -187,6 +188,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | +| [kubernetes](#provider\_kubernetes) | >= 2.10 | | [tls](#provider\_tls) | >= 3.0 | ## Modules @@ -216,6 +218,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_security_group.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group_rule.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | +| [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -226,6 +230,12 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [attach\_cluster\_encryption\_policy](#input\_attach\_cluster\_encryption\_policy) | Indicates whether or not to attach an additional policy for the cluster IAM role to utilize the encryption key provided | `bool` | `true` | no | +| [aws\_auth\_accounts](#input\_aws\_auth\_accounts) | List of account maps to add to the aws-auth configmap | `list(any)` | `[]` | no | +| [aws\_auth\_fargate\_profile\_pod\_execution\_role\_arns](#input\_aws\_auth\_fargate\_profile\_pod\_execution\_role\_arns) | List of Fargate profile pod execution role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no | +| [aws\_auth\_node\_iam\_role\_arns\_non\_windows](#input\_aws\_auth\_node\_iam\_role\_arns\_non\_windows) | List of non-Windows based node IAM role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no | +| [aws\_auth\_node\_iam\_role\_arns\_windows](#input\_aws\_auth\_node\_iam\_role\_arns\_windows) | List of Windows based node IAM role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no | +| [aws\_auth\_roles](#input\_aws\_auth\_roles) | List of role maps to add to the aws-auth configmap | `list(any)` | `[]` | no | +| [aws\_auth\_users](#input\_aws\_auth\_users) | List of user maps to add to the aws-auth configmap | `list(any)` | `[]` | no | | [cloudwatch\_log\_group\_kms\_key\_id](#input\_cloudwatch\_log\_group\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `null` | no | | [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days | `number` | `90` | no | | [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | @@ -255,6 +265,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.21`) | `string` | `null` | no | | [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | +| [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the confgimap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | | [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster or use the existing `cluster_security_group_id` | `bool` | `true` | no | | [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy) | `bool` | `false` | no | @@ -274,6 +285,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | +| [manage\_aws\_auth\_configmap](#input\_manage\_aws\_auth\_configmap) | Determines whether to manage the aws-auth configmap | `bool` | `false` | no | | [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no | | [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | | [node\_security\_group\_id](#input\_node\_security\_group\_id) | ID of an existing security group to attach to the node groups created | `string` | `""` | no | @@ -293,7 +305,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Description | |------|-------------| -| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | [DEPRECATED - use `var.manage_aws_auth_configmap`] Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | | [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | | [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | diff --git a/examples/complete/README.md b/examples/complete/README.md index a432ec9c47..d329fad1d0 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -35,14 +35,13 @@ Note that this example may create resources which cost money. Run `terraform des |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | -| [null](#requirement\_null) | >= 3.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | -| [null](#provider\_null) | >= 3.0 | ## Modules @@ -64,8 +63,6 @@ Note that this example may create resources which cost money. Run `terraform des |------|------| | [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [null_resource.patch](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | ## Inputs diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 79fb6c018e..a3589cb3cf 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -8,6 +8,18 @@ provider "aws" { } } +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1alpha1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + } +} + locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" region = "eu-west-1" @@ -180,6 +192,43 @@ module "eks" { } } + # aws-auth configmap + manage_aws_auth_configmap = true + + aws_auth_node_iam_role_arns_non_windows = [ + module.eks_managed_node_group.iam_role_arn, + module.self_managed_node_group.iam_role_arn, + ] + aws_auth_fargate_profile_pod_execution_role_arns = [ + module.fargate_profile.fargate_profile_pod_execution_role_arn + ] + + aws_auth_roles = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] + + aws_auth_users = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] + + aws_auth_accounts = [ + "777777777777", + "888888888888", + ] + tags = local.tags } @@ -269,80 +318,6 @@ module "disabled_self_managed_node_group" { create = false } -################################################################################ -# aws-auth configmap -# Only EKS managed node groups automatically add roles to aws-auth configmap -# so we need to ensure fargate profiles and self-managed node roles are added -################################################################################ - -data "aws_eks_cluster_auth" "this" { - name = module.eks.cluster_id -} - -locals { - kubeconfig = yamlencode({ - apiVersion = "v1" - kind = "Config" - current-context = "terraform" - clusters = [{ - name = module.eks.cluster_id - cluster = { - certificate-authority-data = module.eks.cluster_certificate_authority_data - server = module.eks.cluster_endpoint - } - }] - contexts = [{ - name = "terraform" - context = { - cluster = module.eks.cluster_id - user = "terraform" - } - }] - users = [{ - name = "terraform" - user = { - token = data.aws_eks_cluster_auth.this.token - } - }] - }) - - # we have to combine the configmap created by the eks module with the externally created node group/profile sub-modules - aws_auth_configmap_yaml = <<-EOT - ${chomp(module.eks.aws_auth_configmap_yaml)} - - rolearn: ${module.eks_managed_node_group.iam_role_arn} - username: system:node:{{EC2PrivateDNSName}} - groups: - - system:bootstrappers - - system:nodes - - rolearn: ${module.self_managed_node_group.iam_role_arn} - username: system:node:{{EC2PrivateDNSName}} - groups: - - system:bootstrappers - - system:nodes - - rolearn: ${module.fargate_profile.fargate_profile_pod_execution_role_arn} - username: system:node:{{SessionName}} - groups: - - system:bootstrappers - - system:nodes - - system:node-proxier - EOT -} - -resource "null_resource" "patch" { - triggers = { - kubeconfig = base64encode(local.kubeconfig) - cmd_patch = "kubectl patch configmap/aws-auth --patch \"${local.aws_auth_configmap_yaml}\" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode)" - } - - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - environment = { - KUBECONFIG = self.triggers.kubeconfig - } - command = self.triggers.cmd_patch - } -} - ################################################################################ # Supporting resources ################################################################################ diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index a53f5a3d28..6d6dc45be6 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -6,9 +6,9 @@ terraform { source = "hashicorp/aws" version = ">= 3.72" } - null = { - source = "hashicorp/null" - version = ">= 3.0" + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" } } } diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 33dd2c6ebf..5293cd5232 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -58,7 +58,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | -| [null](#requirement\_null) | >= 3.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | | [tls](#requirement\_tls) | >= 2.2 | ## Providers @@ -66,7 +66,6 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | -| [null](#provider\_null) | >= 3.0 | | [tls](#provider\_tls) | >= 2.2 | ## Modules @@ -89,13 +88,11 @@ Note that this example may create resources which cost money. Run `terraform des | [aws_launch_template.external](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.remote_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [null_resource.patch](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_ami.eks_default_arm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | ## Inputs diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 8fc40db12f..3b988feef3 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -2,6 +2,18 @@ provider "aws" { region = local.region } +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1alpha1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + } +} + locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" cluster_version = "1.22" @@ -58,6 +70,8 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets + manage_aws_auth_configmap = true + # Extend cluster security group rules cluster_security_group_additional_rules = { egress_nodes_ephemeral_ports_tcp = { @@ -340,59 +354,6 @@ resource "aws_iam_role_policy_attachment" "additional" { role = each.value.iam_role_name } -################################################################################ -# aws-auth configmap -# Only EKS managed node groups automatically add roles to aws-auth configmap -# so we need to ensure fargate profiles and self-managed node roles are added -################################################################################ - -data "aws_eks_cluster_auth" "this" { - name = module.eks.cluster_id -} - -locals { - kubeconfig = yamlencode({ - apiVersion = "v1" - kind = "Config" - current-context = "terraform" - clusters = [{ - name = module.eks.cluster_id - cluster = { - certificate-authority-data = module.eks.cluster_certificate_authority_data - server = module.eks.cluster_endpoint - } - }] - contexts = [{ - name = "terraform" - context = { - cluster = module.eks.cluster_id - user = "terraform" - } - }] - users = [{ - name = "terraform" - user = { - token = data.aws_eks_cluster_auth.this.token - } - }] - }) -} - -resource "null_resource" "patch" { - triggers = { - kubeconfig = base64encode(local.kubeconfig) - cmd_patch = "kubectl patch configmap/aws-auth --patch \"${module.eks.aws_auth_configmap_yaml}\" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode)" - } - - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - environment = { - KUBECONFIG = self.triggers.kubeconfig - } - command = self.triggers.cmd_patch - } -} - ################################################################################ # Supporting Resources ################################################################################ diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index 2ba2b42d4e..40b7e32ab9 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -6,13 +6,13 @@ terraform { source = "hashicorp/aws" version = ">= 3.72" } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } tls = { source = "hashicorp/tls" version = ">= 2.2" } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } } } diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index cec6ef345b..1dcb8638c8 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -21,6 +21,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index 22e8d7265f..6d6dc45be6 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -6,5 +6,9 @@ terraform { source = "hashicorp/aws" version = ">= 3.72" } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } } } diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 3798d8e17e..d668611bd7 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -27,7 +27,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | -| [null](#requirement\_null) | >= 3.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | | [tls](#requirement\_tls) | >= 2.2 | ## Providers @@ -35,7 +35,6 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | -| [null](#provider\_null) | >= 3.0 | | [tls](#provider\_tls) | >= 2.2 | ## Modules @@ -54,12 +53,10 @@ Note that this example may create resources which cost money. Run `terraform des | [aws_kms_key.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [null_resource.apply](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | ## Inputs diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index a7ad36cc81..e1f7caf3a9 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -2,6 +2,18 @@ provider "aws" { region = local.region } +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1alpha1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + } +} + locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" cluster_version = "1.22" @@ -46,6 +58,10 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets + # Self managed node groups will not automatically create the aws-auth configmap so we need to + create_aws_auth_configmap = true + manage_aws_auth_configmap = true + # Extend cluster security group rules cluster_security_group_additional_rules = { egress_nodes_ephemeral_ports_tcp = { @@ -291,62 +307,6 @@ module "eks" { tags = local.tags } -################################################################################ -# aws-auth configmap -# Only EKS managed node groups automatically add roles to aws-auth configmap -# so we need to ensure fargate profiles and self-managed node roles are added -################################################################################ - -data "aws_eks_cluster_auth" "this" { - name = module.eks.cluster_id -} - -locals { - kubeconfig = yamlencode({ - apiVersion = "v1" - kind = "Config" - current-context = "terraform" - clusters = [{ - name = module.eks.cluster_id - cluster = { - certificate-authority-data = module.eks.cluster_certificate_authority_data - server = module.eks.cluster_endpoint - } - }] - contexts = [{ - name = "terraform" - context = { - cluster = module.eks.cluster_id - user = "terraform" - } - }] - users = [{ - name = "terraform" - user = { - token = data.aws_eks_cluster_auth.this.token - } - }] - }) -} - -resource "null_resource" "apply" { - triggers = { - kubeconfig = base64encode(local.kubeconfig) - cmd_patch = <<-EOT - kubectl create configmap aws-auth -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) - kubectl patch configmap/aws-auth --patch "${module.eks.aws_auth_configmap_yaml}" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) - EOT - } - - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - environment = { - KUBECONFIG = self.triggers.kubeconfig - } - command = self.triggers.cmd_patch - } -} - ################################################################################ # Supporting Resources ################################################################################ diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index 2ba2b42d4e..40b7e32ab9 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -6,13 +6,13 @@ terraform { source = "hashicorp/aws" version = ">= 3.72" } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } tls = { source = "hashicorp/tls" version = ">= 2.2" } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } } } diff --git a/main.tf b/main.tf index 0e802d82f8..bcd864f20b 100644 --- a/main.tf +++ b/main.tf @@ -347,3 +347,98 @@ resource "aws_eks_identity_provider_config" "this" { tags = var.tags } + +################################################################################ +# aws-auth configmap +################################################################################ + +locals { + node_iam_role_arns_non_windows = compact(concat( + [for group in module.eks_managed_node_group : group.iam_role_arn], + [for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"], + var.aws_auth_node_iam_role_arns_non_windows, + )) + + node_iam_role_arns_windows = compact(concat( + [for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"], + var.aws_auth_node_iam_role_arns_windows, + )) + + fargate_profile_pod_execution_role_arns = compact(concat( + [for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn], + var.aws_auth_fargate_profile_pod_execution_role_arns, + )) + + aws_auth_configmap_data = { + mapRoles = yamlencode(concat( + [for role_arn in local.node_iam_role_arns_non_windows : { + rolearn = role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + } + ], + [for role_arn in local.node_iam_role_arns_windows : { + rolearn = role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "eks:kube-proxy-windows", + "system:bootstrappers", + "system:nodes", + ] + } + ], + # Fargate profile + [for role_arn in local.fargate_profile_pod_execution_role_arns : { + rolearn = role_arn + username = "system:node:{{SessionName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + "system:node-proxier", + ] + } + ], + var.aws_auth_roles + )) + mapUsers = yamlencode(var.aws_auth_users) + mapAccounts = yamlencode(var.aws_auth_accounts) + } +} + +resource "kubernetes_config_map" "aws_auth" { + count = var.create && var.create_aws_auth_configmap ? 1 : 0 + + metadata { + name = "aws-auth" + namespace = "kube-system" + } + + data = local.aws_auth_configmap_data + + lifecycle { + # We are ignoring the data here since we will manage it with the resource below + # This is only intended to be used in scenarios where the configmap does not exist + ignore_changes = [data] + } +} + +resource "kubernetes_config_map_v1_data" "aws_auth" { + count = var.create && var.manage_aws_auth_configmap ? 1 : 0 + + force = true + + metadata { + name = "aws-auth" + namespace = "kube-system" + } + + data = local.aws_auth_configmap_data + + depends_on = [ + # Required for instances where the configmap does not exist yet to avoid race condition + kubernetes_config_map.aws_auth, + ] +} diff --git a/outputs.tf b/outputs.tf index 03eede315b..747d30d7fe 100644 --- a/outputs.tf +++ b/outputs.tf @@ -182,7 +182,7 @@ output "self_managed_node_groups_autoscaling_group_names" { ################################################################################ output "aws_auth_configmap_yaml" { - description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" + description = "[DEPRECATED - use `var.manage_aws_auth_configmap`] Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" value = templatefile("${path.module}/templates/aws_auth_cm.tpl", { eks_managed_role_arns = [for group in module.eks_managed_node_group : group.iam_role_arn] diff --git a/variables.tf b/variables.tf index ed972387a5..51cd3f41d2 100644 --- a/variables.tf +++ b/variables.tf @@ -428,3 +428,55 @@ variable "putin_khuylo" { type = bool default = true } + +################################################################################ +# aws-auth configmap +################################################################################ + +variable "manage_aws_auth_configmap" { + description = "Determines whether to manage the aws-auth configmap" + type = bool + default = false +} + +variable "create_aws_auth_configmap" { + description = "Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the confgimap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap`" + type = bool + default = false +} + +variable "aws_auth_node_iam_role_arns_non_windows" { + description = "List of non-Windows based node IAM role ARNs to add to the aws-auth configmap" + type = list(string) + default = [] +} + +variable "aws_auth_node_iam_role_arns_windows" { + description = "List of Windows based node IAM role ARNs to add to the aws-auth configmap" + type = list(string) + default = [] +} + +variable "aws_auth_fargate_profile_pod_execution_role_arns" { + description = "List of Fargate profile pod execution role ARNs to add to the aws-auth configmap" + type = list(string) + default = [] +} + +variable "aws_auth_roles" { + description = "List of role maps to add to the aws-auth configmap" + type = list(any) + default = [] +} + +variable "aws_auth_users" { + description = "List of user maps to add to the aws-auth configmap" + type = list(any) + default = [] +} + +variable "aws_auth_accounts" { + description = "List of account maps to add to the aws-auth configmap" + type = list(any) + default = [] +} diff --git a/versions.tf b/versions.tf index 85e4ed7915..fde7af0f23 100644 --- a/versions.tf +++ b/versions.tf @@ -10,5 +10,9 @@ terraform { source = "hashicorp/tls" version = ">= 3.0" } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } } } From 19b52d24fc62cf551a6cc45937fbf3902ccd1789 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 9 Apr 2022 07:16:21 +0000 Subject: [PATCH 530/962] chore(release): version 18.20.0 [skip ci] ## [18.20.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.19.0...v18.20.0) (2022-04-09) ### Features * Add support for managing `aws-auth` configmap using new `kubernetes_config_map_v1_data` resource ([#1999](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1999)) ([da3d54c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/da3d54cde70adfd8b5d2770805b17d526923113e)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 602f9074df..3dd1fb7ce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.20.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.19.0...v18.20.0) (2022-04-09) + + +### Features + +* Add support for managing `aws-auth` configmap using new `kubernetes_config_map_v1_data` resource ([#1999](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1999)) ([da3d54c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/da3d54cde70adfd8b5d2770805b17d526923113e)) + ## [18.19.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.18.0...v18.19.0) (2022-04-04) From c576aadce968d09f3295fc06f0766cc9e2a35e29 Mon Sep 17 00:00:00 2001 From: Ken Kaizu Date: Sat, 9 Apr 2022 21:26:44 +0900 Subject: [PATCH 531/962] fix: iam_role_user_name_prefix type as an bool (#2000) --- README.md | 2 +- modules/eks-managed-node-group/README.md | 2 +- modules/eks-managed-node-group/variables.tf | 2 +- modules/fargate-profile/README.md | 2 +- modules/fargate-profile/variables.tf | 2 +- modules/self-managed-node-group/README.md | 2 +- modules/self-managed-node-group/variables.tf | 2 +- variables.tf | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a98eb30712..6ad9228198 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [iam\_role\_path](#input\_iam\_role\_path) | Cluster IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | -| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [manage\_aws\_auth\_configmap](#input\_manage\_aws\_auth\_configmap) | Determines whether to manage the aws-auth configmap | `bool` | `false` | no | | [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no | | [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 6c02fe5974..0733ae8642 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -120,7 +120,7 @@ module "eks_managed_node_group" { | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | -| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `null` | no | | [instance\_types](#input\_instance\_types) | Set of instance types associated with the EKS Node Group. Defaults to `["t3.medium"]` | `list(string)` | `null` | no | | [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 93e9555f67..5929817887 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -450,7 +450,7 @@ variable "iam_role_name" { variable "iam_role_use_name_prefix" { description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" - type = string + type = bool default = true } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 42349dd2a3..97dd0d3e53 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -68,7 +68,7 @@ No modules. | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | -| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [name](#input\_name) | Name of the EKS Fargate Profile | `string` | `""` | no | | [selectors](#input\_selectors) | Configuration block(s) for selecting Kubernetes Pods to execute with this Fargate Profile | `any` | `[]` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs for the EKS Fargate Profile | `list(string)` | `[]` | no | diff --git a/modules/fargate-profile/variables.tf b/modules/fargate-profile/variables.tf index 5accf35e1b..4d5e95ccbe 100644 --- a/modules/fargate-profile/variables.tf +++ b/modules/fargate-profile/variables.tf @@ -40,7 +40,7 @@ variable "iam_role_name" { variable "iam_role_use_name_prefix" { description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" - type = string + type = bool default = true } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 8d5f1bb2c0..af37888d34 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -118,7 +118,7 @@ module "self_managed_node_group" { | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | -| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix | `string` | `true` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [initial\_lifecycle\_hooks](#input\_initial\_lifecycle\_hooks) | One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource | `list(map(string))` | `[]` | no | | [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | | [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `null` | no | diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index c261cdcdb5..c7e04f30d5 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -556,7 +556,7 @@ variable "iam_role_name" { variable "iam_role_use_name_prefix" { description = "Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix" - type = string + type = bool default = true } diff --git a/variables.tf b/variables.tf index 51cd3f41d2..c5337676fb 100644 --- a/variables.tf +++ b/variables.tf @@ -283,7 +283,7 @@ variable "iam_role_name" { variable "iam_role_use_name_prefix" { description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" - type = string + type = bool default = true } From 96cb4544223d2abb4ab73cc4f90f92d65bb12bb1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 9 Apr 2022 12:27:19 +0000 Subject: [PATCH 532/962] chore(release): version 18.20.1 [skip ci] ### [18.20.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.0...v18.20.1) (2022-04-09) ### Bug Fixes * iam_role_user_name_prefix type as an bool ([#2000](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2000)) ([c576aad](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c576aadce968d09f3295fc06f0766cc9e2a35e29)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dd1fb7ce7..60fed0f959 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.20.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.0...v18.20.1) (2022-04-09) + + +### Bug Fixes + +* iam_role_user_name_prefix type as an bool ([#2000](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2000)) ([c576aad](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c576aadce968d09f3295fc06f0766cc9e2a35e29)) + ## [18.20.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.19.0...v18.20.0) (2022-04-09) From cd8731ec5a659fa7e3f31555bc558e4fd3dedc5d Mon Sep 17 00:00:00 2001 From: yutachaos <18604471+yutachaos@users.noreply.github.com> Date: Sun, 10 Apr 2022 02:26:15 +0900 Subject: [PATCH 533/962] chore: Fixed doc link (#1984) Co-authored-by: Anton Babenko --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ad9228198..32e47fb98c 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Some of the addon/controller policies that are currently supported include: - [EBS CSI Driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/example-iam-policy.json) - [VPC CNI](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html) - [Node Termination Handler](https://github.com/aws/aws-node-termination-handler#5-create-an-iam-role-for-the-pods) -- [Karpenter](https://github.com/aws/karpenter/blob/main/website/content/en/preview/getting-started/cloudformation.yaml) +- [Karpenter](https://karpenter.sh/preview/getting-started/getting-started-with-terraform/#create-the-karpentercontroller-iam-role) - [Load Balancer Controller](https://github.com/kubernetes-sigs/aws-load-balancer-controller/blob/main/docs/install/iam_policy.json) See [terraform-aws-iam/modules/iam-role-for-service-accounts](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) for current list of supported addon/controller policies as more are added to the project. From 69a815c7dfe3c44feddfaa129c36953656d5f123 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 10 Apr 2022 09:20:38 -0400 Subject: [PATCH 534/962] docs: Update documentation to reflect latest changes related to aws-auth configmap and upgrading (#2003) --- README.md | 31 ++++++++++++++++++++++++++++++- docs/UPGRADE-17.0.md | 4 ++-- docs/UPGRADE-18.0.md | 11 +++++++++++ docs/faq.md | 11 ----------- variables.tf | 2 +- 5 files changed, 44 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 32e47fb98c..c013ef201c 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,35 @@ module "eks" { } } + # aws-auth configmap + manage_aws_auth_configmap = true + + aws_auth_roles = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] + + aws_auth_users = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] + + aws_auth_accounts = [ + "777777777777", + "888888888888", + ] + tags = { Environment = "dev" Terraform = "true" @@ -265,7 +294,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.21`) | `string` | `null` | no | | [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | -| [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the confgimap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | +| [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | | [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster or use the existing `cluster_security_group_id` | `bool` | `true` | no | | [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy) | `bool` | `false` | no | diff --git a/docs/UPGRADE-17.0.md b/docs/UPGRADE-17.0.md index d078bf9e6a..2511cf4b2d 100644 --- a/docs/UPGRADE-17.0.md +++ b/docs/UPGRADE-17.0.md @@ -34,7 +34,7 @@ module "eks" { } ``` -4. Run `terraform plan`, you shoud see that only `random_pets` will be destroyed +4. Run `terraform plan`, you should see that only `random_pets` will be destroyed ```shell Terraform will perform the following actions: @@ -62,4 +62,4 @@ Plan: 0 to add, 0 to change, 1 to destroy. 5. If everything sounds good to you, run `terraform apply` -After the first apply, we recommand you to create a new node group and let the module use the `node_group_name_prefix` (by removing the `name` argument) to generate names and avoid collision during node groups re-creation if needed, because the lifce cycle is `create_before_destroy = true`. +After the first apply, we recommend you to create a new node group and let the module use the `node_group_name_prefix` (by removing the `name` argument) to generate names and avoid collision during node groups re-creation if needed, because the lifecycle is `create_before_destroy = true`. diff --git a/docs/UPGRADE-18.0.md b/docs/UPGRADE-18.0.md index 33c1767be5..b8458e2d1b 100644 --- a/docs/UPGRADE-18.0.md +++ b/docs/UPGRADE-18.0.md @@ -4,6 +4,17 @@ Please consult the `examples` directory for reference example configurations. If Note: please see https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1744 where users have shared the steps/changes that have worked for their configurations to upgrade. Due to the numerous configuration possibilities, it is difficult to capture specific steps that will work for all; this has proven to be a useful thread to share collective information from the broader community regarding v18.x upgrades. +For most users, adding the following to your v17.x configuration will preserve the state of your cluster control plane when upgrading to v18.x: + +```hcl +prefix_separator = "" +iam_role_name = $CLUSTER_NAME +cluster_security_group_name = $CLUSTER_NAME +cluster_security_group_description = "EKS cluster security group." +``` + +See more information [here](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1744#issuecomment-1027359982) + ## List of backwards incompatible changes - Launch configuration support has been removed and only launch template is supported going forward. AWS is no longer adding new features back into launch configuration and their docs state [`We strongly recommend that you do not use launch configurations. They do not provide full functionality for Amazon EC2 Auto Scaling or Amazon EC2. We provide information about launch configurations for customers who have not yet migrated from launch configurations to launch templates.`](https://docs.aws.amazon.com/autoscaling/ec2/userguide/LaunchConfiguration.html) diff --git a/docs/faq.md b/docs/faq.md index 33e8ad7964..7a52410715 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,22 +1,11 @@ # Frequently Asked Questions -- [How do I manage the `aws-auth` configmap?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-do-i-manage-the-aws-auth-configmap) - [I received an error: `Error: Invalid for_each argument ...`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#i-received-an-error-error-invalid-for_each-argument-) - [Why are nodes not being registered?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-nodes-not-being-registered) - [Why are there no changes when a node group's `desired_size` is modified?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-there-no-changes-when-a-node-groups-desired_size-is-modified) - [How can I deploy Windows based nodes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-can-i-deploy-windows-based-nodes) - [How do I access compute resource attributes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-do-i-access-compute-resource-attributes) -### How do I manage the `aws-auth` configmap? - -TL;DR - https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1901 - -- Users can roll their own equivalent of `kubectl patch ...` using the [`null_resource`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/9a99689cc13147f4afc426b34ba009875a28614e/examples/complete/main.tf#L301-L336) -- There is a module that was created to fill this gap that provides a Kubernetes based approach to provision: https://github.com/aidanmelen/terraform-aws-eks-auth -- Ideally, one of the following issues are resolved upstream for a more native experience for users: - - https://github.com/aws/containers-roadmap/issues/185 - - https://github.com/hashicorp/terraform-provider-kubernetes/issues/723 - ### I received an error: `Error: Invalid for_each argument ...` Users may encounter an error such as `Error: Invalid for_each argument - The "for_each" value depends on resource attributes that cannot be determined until apply, so Terraform cannot predict how many instances will be created. To work around this, use the -target argument to first apply ...` diff --git a/variables.tf b/variables.tf index c5337676fb..d78fb251ec 100644 --- a/variables.tf +++ b/variables.tf @@ -440,7 +440,7 @@ variable "manage_aws_auth_configmap" { } variable "create_aws_auth_configmap" { - description = "Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the confgimap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap`" + description = "Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap`" type = bool default = false } From b5ae5daa39f8380dc21c9ef1daff22242930692e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 12 Apr 2022 04:36:03 -0400 Subject: [PATCH 535/962] fix: Avoid re-naming the primary security group through a `Name` tag and leave to the EKS service to manage (#2010) --- examples/eks_managed_node_group/main.tf | 7 +++++++ main.tf | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 3b988feef3..308f05a340 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -67,6 +67,13 @@ module "eks" { resources = ["secrets"] }] + cluster_tags = { + # This should not affect the name of the cluster primary security group + # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2006 + # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2008 + Name = local.name + } + vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets diff --git a/main.tf b/main.tf index bcd864f20b..9e7d081b56 100644 --- a/main.tf +++ b/main.tf @@ -60,7 +60,10 @@ resource "aws_eks_cluster" "this" { } resource "aws_ec2_tag" "cluster_primary_security_group" { - for_each = { for k, v in merge(var.tags, var.cluster_tags) : k => v if local.create } + # This should not affect the name of the cluster primary security group + # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2006 + # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2008 + for_each = { for k, v in merge(var.tags, var.cluster_tags) : k => v if local.create && k != "Name" } resource_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id key = each.key From 34ecad6e1e373e68d786119082a48fe02f2a4b0a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 12 Apr 2022 08:36:43 +0000 Subject: [PATCH 536/962] chore(release): version 18.20.2 [skip ci] ### [18.20.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.1...v18.20.2) (2022-04-12) ### Bug Fixes * Avoid re-naming the primary security group through a `Name` tag and leave to the EKS service to manage ([#2010](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2010)) ([b5ae5da](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b5ae5daa39f8380dc21c9ef1daff22242930692e)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60fed0f959..46e8716ff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.20.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.1...v18.20.2) (2022-04-12) + + +### Bug Fixes + +* Avoid re-naming the primary security group through a `Name` tag and leave to the EKS service to manage ([#2010](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2010)) ([b5ae5da](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b5ae5daa39f8380dc21c9ef1daff22242930692e)) + ### [18.20.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.0...v18.20.1) (2022-04-09) From 681e00aafea093be72ec06ada3825a23a181b1c5 Mon Sep 17 00:00:00 2001 From: Hugues Alary Date: Fri, 15 Apr 2022 00:23:00 -0700 Subject: [PATCH 537/962] docs: Update UPGRADE-18.0.md (#2014) --- docs/UPGRADE-18.0.md | 157 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/docs/UPGRADE-18.0.md b/docs/UPGRADE-18.0.md index b8458e2d1b..3bfb672c3f 100644 --- a/docs/UPGRADE-18.0.md +++ b/docs/UPGRADE-18.0.md @@ -565,6 +565,163 @@ module "cluster_after" { } ``` +### Diff of before <> after + +```diff + module "eks" { + source = "terraform-aws-modules/eks/aws" +- version = "~> 17.0" ++ version = "~> 18.0" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id +- subnets = module.vpc.private_subnets ++ subnet_ids = module.vpc.private_subnets + +- # Managed Node Groups +- node_groups_defaults = { ++ eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + disk_size = 50 + } + +- node_groups = { ++ eks_managed_node_groups = { + node_group = { +- min_capacity = 1 +- max_capacity = 10 +- desired_capacity = 1 ++ min_size = 1 ++ max_size = 10 ++ desired_size = 1 + + instance_types = ["t3.large"] + capacity_type = "SPOT" + + update_config = { + max_unavailable_percentage = 50 + } + +- k8s_labels = { ++ labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + taints = [ + { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + ] + +- additional_tags = { ++ tags = { + ExtraTag = "example" + } + } + } + +- # Worker groups +- worker_additional_security_group_ids = [aws_security_group.additional.id] +- +- worker_groups_launch_template = [ +- { +- name = "worker-group" +- override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] +- spot_instance_pools = 4 +- asg_max_size = 5 +- asg_desired_capacity = 2 +- kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" +- public_ip = true +- }, +- ] ++ self_managed_node_group_defaults = { ++ vpc_security_group_ids = [aws_security_group.additional.id] ++ } ++ ++ self_managed_node_groups = { ++ worker_group = { ++ name = "worker-group" ++ ++ min_size = 1 ++ max_size = 5 ++ desired_size = 2 ++ instance_type = "m4.large" ++ ++ bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" ++ ++ block_device_mappings = { ++ xvda = { ++ device_name = "/dev/xvda" ++ ebs = { ++ delete_on_termination = true ++ encrypted = false ++ volume_size = 100 ++ volume_type = "gp2" ++ } ++ ++ } ++ } ++ ++ use_mixed_instances_policy = true ++ mixed_instances_policy = { ++ instances_distribution = { ++ spot_instance_pools = 4 ++ } ++ ++ override = [ ++ { instance_type = "m5.large" }, ++ { instance_type = "m5a.large" }, ++ { instance_type = "m5d.large" }, ++ { instance_type = "m5ad.large" }, ++ ] ++ } ++ } ++ } + + # Fargate + fargate_profiles = { + default = { + name = "default" + selectors = [ + { + namespace = "kube-system" + labels = { + k8s-app = "kube-dns" + } + }, + { + namespace = "default" + } + ] + + tags = { + Owner = "test" + } + + timeouts = { + create = "20m" + delete = "20m" + } + } + } + + tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + } + +``` + ### Attaching an IAM role policy to a Fargate profile #### Before 17.x From c173ba2d62d228729fe6c68f713af6dbe15e7233 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 20 Apr 2022 12:14:09 -0400 Subject: [PATCH 538/962] fix: Add `compact()` to `aws_auth_configmap_yaml` for when node groups are set to `create = false` (#2029) --- outputs.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/outputs.tf b/outputs.tf index 747d30d7fe..7a06218d03 100644 --- a/outputs.tf +++ b/outputs.tf @@ -185,10 +185,10 @@ output "aws_auth_configmap_yaml" { description = "[DEPRECATED - use `var.manage_aws_auth_configmap`] Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" value = templatefile("${path.module}/templates/aws_auth_cm.tpl", { - eks_managed_role_arns = [for group in module.eks_managed_node_group : group.iam_role_arn] - self_managed_role_arns = [for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"] - win32_self_managed_role_arns = [for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"] - fargate_profile_pod_execution_role_arns = [for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn] + eks_managed_role_arns = compact([for group in module.eks_managed_node_group : group.iam_role_arn]) + self_managed_role_arns = compact([for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"]) + win32_self_managed_role_arns = compact([for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"]) + fargate_profile_pod_execution_role_arns = compact([for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn]) } ) } From 70b0cff56906e4b75f96d761782040b19350c4e0 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 20 Apr 2022 16:14:50 +0000 Subject: [PATCH 539/962] chore(release): version 18.20.3 [skip ci] ### [18.20.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.2...v18.20.3) (2022-04-20) ### Bug Fixes * Add `compact()` to `aws_auth_configmap_yaml` for when node groups are set to `create = false` ([#2029](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2029)) ([c173ba2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c173ba2d62d228729fe6c68f713af6dbe15e7233)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46e8716ff2..e410ba364a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.20.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.2...v18.20.3) (2022-04-20) + + +### Bug Fixes + +* Add `compact()` to `aws_auth_configmap_yaml` for when node groups are set to `create = false` ([#2029](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2029)) ([c173ba2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c173ba2d62d228729fe6c68f713af6dbe15e7233)) + ### [18.20.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.1...v18.20.2) (2022-04-12) From 5da692df67cae313711e94216949d1105da6a87f Mon Sep 17 00:00:00 2001 From: Xin Chen Date: Thu, 21 Apr 2022 01:44:48 +0800 Subject: [PATCH 540/962] fix: Correct DNS suffix for OIDC provider (#2026) Co-authored-by: Xin Chen Co-authored-by: Bryant Biggs --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 9e7d081b56..61ae843190 100644 --- a/main.tf +++ b/main.tf @@ -174,7 +174,7 @@ data "tls_certificate" "this" { resource "aws_iam_openid_connect_provider" "oidc_provider" { count = local.create && var.enable_irsa ? 1 : 0 - client_id_list = distinct(compact(concat(["sts.${data.aws_partition.current.dns_suffix}"], var.openid_connect_audiences))) + client_id_list = distinct(compact(concat(["sts.${local.dns_suffix}"], var.openid_connect_audiences))) thumbprint_list = concat([data.tls_certificate.this[0].certificates[0].sha1_fingerprint], var.custom_oidc_thumbprints) url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer From 5815d17cd2330286250ea4f81fcad3ac9bf19776 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 20 Apr 2022 17:45:25 +0000 Subject: [PATCH 541/962] chore(release): version 18.20.4 [skip ci] ### [18.20.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.3...v18.20.4) (2022-04-20) ### Bug Fixes * Correct DNS suffix for OIDC provider ([#2026](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2026)) ([5da692d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5da692df67cae313711e94216949d1105da6a87f)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e410ba364a..85dbba4c7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.20.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.3...v18.20.4) (2022-04-20) + + +### Bug Fixes + +* Correct DNS suffix for OIDC provider ([#2026](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2026)) ([5da692d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5da692df67cae313711e94216949d1105da6a87f)) + ### [18.20.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.2...v18.20.3) (2022-04-20) From 51e418216f210647b69bbd06e569a061c2f0e3c1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 21 Apr 2022 11:34:58 -0400 Subject: [PATCH 542/962] fix: Add conditional variable to allow users to opt out of tagging cluster primary security group (#2034) --- .pre-commit-config.yaml | 2 +- README.md | 1 + examples/complete/main.tf | 1 - examples/eks_managed_node_group/main.tf | 4 ++-- examples/self_managed_node_group/main.tf | 3 +-- main.tf | 2 +- variables.tf | 6 ++++++ 7 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ba45548ddd..a0925b0080 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.64.1 + rev: v1.68.1 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/README.md b/README.md index c013ef201c..db840bff74 100644 --- a/README.md +++ b/README.md @@ -296,6 +296,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | +| [create\_cluster\_primary\_security\_group\_tags](#input\_create\_cluster\_primary\_security\_group\_tags) | Indicates whether or not to tag the cluster's primary security group. This security group is created by the EKS service, not the module, and therefore tagging is handled after cluster creation | `bool` | `true` | no | | [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster or use the existing `cluster_security_group_id` | `bool` | `true` | no | | [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy) | `bool` | `false` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether a an IAM role is created or to use an existing IAM role | `bool` | `true` | no | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index a3589cb3cf..3702dd1b55 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -125,7 +125,6 @@ module "eks" { # EKS Managed Node Group(s) eks_managed_node_group_defaults = { ami_type = "AL2_x86_64" - disk_size = 50 instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] attach_cluster_primary_security_group = true diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 308f05a340..941509afde 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -114,7 +114,6 @@ module "eks" { eks_managed_node_group_defaults = { ami_type = "AL2_x86_64" - disk_size = 50 instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] # We are using the IRSA created below for permissions @@ -133,6 +132,8 @@ module "eks" { create_launch_template = false launch_template_name = "" + disk_size = 50 + # Remote access cannot be specified with a launch template remote_access = { ec2_ssh_key = aws_key_pair.this.key_name @@ -254,7 +255,6 @@ module "eks" { EOT capacity_type = "SPOT" - disk_size = 256 force_update_version = true instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] labels = { diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index e1f7caf3a9..eb256555d5 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -96,7 +96,7 @@ module "eks" { } self_managed_node_group_defaults = { - disk_size = 50 + create_security_group = false } self_managed_node_groups = { @@ -216,7 +216,6 @@ module "eks" { echo "you are free little kubelet!" EOT - disk_size = 256 instance_type = "m6i.large" launch_template_name = "self-managed-ex" diff --git a/main.tf b/main.tf index 61ae843190..da4edb61b7 100644 --- a/main.tf +++ b/main.tf @@ -63,7 +63,7 @@ resource "aws_ec2_tag" "cluster_primary_security_group" { # This should not affect the name of the cluster primary security group # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2006 # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2008 - for_each = { for k, v in merge(var.tags, var.cluster_tags) : k => v if local.create && k != "Name" } + for_each = { for k, v in merge(var.tags, var.cluster_tags) : k => v if local.create && k != "Name" && var.create_cluster_primary_security_group_tags } resource_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id key = each.key diff --git a/variables.tf b/variables.tf index d78fb251ec..2890834184 100644 --- a/variables.tf +++ b/variables.tf @@ -101,6 +101,12 @@ variable "cluster_tags" { default = {} } +variable "create_cluster_primary_security_group_tags" { + description = "Indicates whether or not to tag the cluster's primary security group. This security group is created by the EKS service, not the module, and therefore tagging is handled after cluster creation" + type = bool + default = true +} + variable "cluster_timeouts" { description = "Create, update, and delete timeout configurations for the cluster" type = map(string) From dc8a6eecddc2c6957ba309a939ee46f39b946461 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 21 Apr 2022 15:35:42 +0000 Subject: [PATCH 543/962] chore(release): version 18.20.5 [skip ci] ### [18.20.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.4...v18.20.5) (2022-04-21) ### Bug Fixes * Add conditional variable to allow users to opt out of tagging cluster primary security group ([#2034](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2034)) ([51e4182](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/51e418216f210647b69bbd06e569a061c2f0e3c1)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85dbba4c7f..487eb152aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.20.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.4...v18.20.5) (2022-04-21) + + +### Bug Fixes + +* Add conditional variable to allow users to opt out of tagging cluster primary security group ([#2034](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2034)) ([51e4182](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/51e418216f210647b69bbd06e569a061c2f0e3c1)) + ### [18.20.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.3...v18.20.4) (2022-04-20) From d969e94bbea89ed616713bdab849f042ceda87ac Mon Sep 17 00:00:00 2001 From: Xin Chen Date: Mon, 25 Apr 2022 18:52:15 +0800 Subject: [PATCH 544/962] chore: Update Karpenter example to use partition data source (#2036) Co-authored-by: Xin Chen --- examples/karpenter/README.md | 3 ++- examples/karpenter/main.tf | 12 +++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 53e6bca5eb..fa06e90170 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -69,7 +69,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| | [eks](#module\_eks) | ../.. | n/a | -| [karpenter\_irsa](#module\_karpenter\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.15 | +| [karpenter\_irsa](#module\_karpenter\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.21.1 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources @@ -80,6 +80,7 @@ Note that this example may create resources which cost money. Run `terraform des | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | ## Inputs diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index fd98f22fbb..4cda4c9c3b 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -2,10 +2,13 @@ provider "aws" { region = local.region } +data "aws_partition" "current" {} + locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" cluster_version = "1.22" region = "eu-west-1" + partition = data.aws_partition.current.partition tags = { Example = local.name @@ -51,7 +54,7 @@ module "eks" { iam_role_additional_policies = [ # Required by Karpenter - "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" ] } } @@ -98,12 +101,15 @@ provider "kubectl" { module "karpenter_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 4.15" + version = "~> 4.21.1" role_name = "karpenter-controller-${local.name}" attach_karpenter_controller_policy = true karpenter_controller_cluster_id = module.eks.cluster_id + karpenter_controller_ssm_parameter_arns = [ + "arn:${local.partition}:ssm:*:*:parameter/aws/service/*" + ] karpenter_controller_node_iam_role_arns = [ module.eks.eks_managed_node_groups["karpenter"].iam_role_arn ] @@ -128,7 +134,7 @@ resource "helm_release" "karpenter" { name = "karpenter" repository = "https://charts.karpenter.sh" chart = "karpenter" - version = "0.8.1" + version = "0.8.2" set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" From 58420b92a0838aa2e17b156b174893b349083a2b Mon Sep 17 00:00:00 2001 From: Seth Pollack Date: Thu, 12 May 2022 14:17:49 -0400 Subject: [PATCH 545/962] feat: Add `create_autoscaling_group` option and extra outputs (#2067) --- modules/self-managed-node-group/README.md | 3 +++ modules/self-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/outputs.tf | 10 ++++++++++ modules/self-managed-node-group/variables.tf | 6 ++++++ node_groups.tf | 2 ++ 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index af37888d34..7c03d9abd7 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -91,6 +91,7 @@ module "self_managed_node_group" { | [cluster\_version](#input\_cluster\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `null` | no | | [create](#input\_create) | Determines whether to create self managed node group or not | `bool` | `true` | no | +| [create\_autoscaling\_group](#input\_create\_autoscaling\_group) | Determines whether to create autoscaling group or not | `bool` | `true` | no | | [create\_iam\_instance\_profile](#input\_create\_iam\_instance\_profile) | Determines whether an IAM instance profile is created or to use an existing IAM instance profile | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create launch template or not | `bool` | `true` | no | | [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | @@ -194,10 +195,12 @@ module "self_managed_node_group" { | [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | | [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | | [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [image\_id](#output\_image\_id) | ID of the image | | [launch\_template\_arn](#output\_launch\_template\_arn) | The ARN of the launch template | | [launch\_template\_id](#output\_launch\_template\_id) | The ID of the launch template | | [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | | [platform](#output\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | | [security\_group\_arn](#output\_security\_group\_arn) | Amazon Resource Name (ARN) of the security group | | [security\_group\_id](#output\_security\_group\_id) | ID of the security group | +| [user\_data](#output\_user\_data) | Base64 encoded user data | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index cb3c70db85..3b59e09299 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -263,7 +263,7 @@ locals { } resource "aws_autoscaling_group" "this" { - count = var.create ? 1 : 0 + count = var.create && var.create_autoscaling_group ? 1 : 0 name = var.use_name_prefix ? null : var.name name_prefix = var.use_name_prefix ? "${var.name}-" : null diff --git a/modules/self-managed-node-group/outputs.tf b/modules/self-managed-node-group/outputs.tf index eaa0c398c8..552e9703be 100644 --- a/modules/self-managed-node-group/outputs.tf +++ b/modules/self-managed-node-group/outputs.tf @@ -145,3 +145,13 @@ output "platform" { description = "Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based" value = var.platform } + +output "image_id" { + description = "ID of the image" + value = try(data.aws_ami.eks_default[0].image_id, "") +} + +output "user_data" { + description = "Base64 encoded user data" + value = try(module.user_data.user_data, "") +} diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index c7e04f30d5..7351038d54 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -266,6 +266,12 @@ variable "launch_template_tags" { # Autoscaling group ################################################################################ +variable "create_autoscaling_group" { + description = "Determines whether to create autoscaling group or not" + type = bool + default = true +} + variable "name" { description = "Name of the Self managed Node Group" type = string diff --git a/node_groups.tf b/node_groups.tf index 09c3747e1d..dba8329919 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -346,6 +346,8 @@ module "self_managed_node_group" { cluster_ip_family = var.cluster_ip_family # Autoscaling Group + create_autoscaling_group = try(each.value.create_autoscaling_group, var.self_managed_node_group_defaults.create_autoscaling_group, true) + name = try(each.value.name, each.key) use_name_prefix = try(each.value.use_name_prefix, var.self_managed_node_group_defaults.use_name_prefix, true) From 7d3c714c5195c5358deb8279e130435241f61271 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 12 May 2022 18:18:27 +0000 Subject: [PATCH 546/962] chore(release): version 18.21.0 [skip ci] ## [18.21.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.5...v18.21.0) (2022-05-12) ### Features * Add `create_autoscaling_group` option and extra outputs ([#2067](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2067)) ([58420b9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/58420b92a0838aa2e17b156b174893b349083a2b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 487eb152aa..10ed7c7bb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.21.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.5...v18.21.0) (2022-05-12) + + +### Features + +* Add `create_autoscaling_group` option and extra outputs ([#2067](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2067)) ([58420b9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/58420b92a0838aa2e17b156b174893b349083a2b)) + ### [18.20.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.4...v18.20.5) (2022-04-21) From 3feb36927f92fb72ab0cfc25a3ab67465872f4bf Mon Sep 17 00:00:00 2001 From: Chris Sng Date: Thu, 2 Jun 2022 21:12:31 +0900 Subject: [PATCH 547/962] feat: Apply `distinct()` on role arns to ensure no duplicated roles in aws-auth configmap (#2097) --- main.tf | 42 +++++++++++++++++++++++++++--------------- outputs.tf | 8 ++++---- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/main.tf b/main.tf index da4edb61b7..9dad2cceee 100644 --- a/main.tf +++ b/main.tf @@ -356,21 +356,33 @@ resource "aws_eks_identity_provider_config" "this" { ################################################################################ locals { - node_iam_role_arns_non_windows = compact(concat( - [for group in module.eks_managed_node_group : group.iam_role_arn], - [for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"], - var.aws_auth_node_iam_role_arns_non_windows, - )) - - node_iam_role_arns_windows = compact(concat( - [for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"], - var.aws_auth_node_iam_role_arns_windows, - )) - - fargate_profile_pod_execution_role_arns = compact(concat( - [for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn], - var.aws_auth_fargate_profile_pod_execution_role_arns, - )) + node_iam_role_arns_non_windows = distinct( + compact( + concat( + [for group in module.eks_managed_node_group : group.iam_role_arn], + [for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"], + var.aws_auth_node_iam_role_arns_non_windows, + ) + ) + ) + + node_iam_role_arns_windows = distinct( + compact( + concat( + [for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"], + var.aws_auth_node_iam_role_arns_windows, + ) + ) + ) + + fargate_profile_pod_execution_role_arns = distinct( + compact( + concat( + [for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn], + var.aws_auth_fargate_profile_pod_execution_role_arns, + ) + ) + ) aws_auth_configmap_data = { mapRoles = yamlencode(concat( diff --git a/outputs.tf b/outputs.tf index 7a06218d03..1245e43777 100644 --- a/outputs.tf +++ b/outputs.tf @@ -185,10 +185,10 @@ output "aws_auth_configmap_yaml" { description = "[DEPRECATED - use `var.manage_aws_auth_configmap`] Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" value = templatefile("${path.module}/templates/aws_auth_cm.tpl", { - eks_managed_role_arns = compact([for group in module.eks_managed_node_group : group.iam_role_arn]) - self_managed_role_arns = compact([for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"]) - win32_self_managed_role_arns = compact([for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"]) - fargate_profile_pod_execution_role_arns = compact([for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn]) + eks_managed_role_arns = distinct(compact([for group in module.eks_managed_node_group : group.iam_role_arn])) + self_managed_role_arns = distinct(compact([for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"])) + win32_self_managed_role_arns = distinct(compact([for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"])) + fargate_profile_pod_execution_role_arns = distinct(compact([for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn])) } ) } From 8178010326242e314535d6ed3b4b6f6c4b465430 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 2 Jun 2022 12:13:08 +0000 Subject: [PATCH 548/962] chore(release): version 18.22.0 [skip ci] ## [18.22.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.21.0...v18.22.0) (2022-06-02) ### Features * Apply `distinct()` on role arns to ensure no duplicated roles in aws-auth configmap ([#2097](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2097)) ([3feb369](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3feb36927f92fb72ab0cfc25a3ab67465872f4bf)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10ed7c7bb3..76d984d8e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.22.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.21.0...v18.22.0) (2022-06-02) + + +### Features + +* Apply `distinct()` on role arns to ensure no duplicated roles in aws-auth configmap ([#2097](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2097)) ([3feb369](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3feb36927f92fb72ab0cfc25a3ab67465872f4bf)) + ## [18.21.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.20.5...v18.21.0) (2022-05-12) From 8584dcb2e0c9061828505c36a8ed8eb6ced02053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20F=C3=A9ron?= Date: Thu, 2 Jun 2022 14:26:49 +0200 Subject: [PATCH 549/962] feat: Add `autoscaling_group_tags` variable to self-managed-node-groups (#2084) Co-authored-by: Bryant Biggs --- examples/self_managed_node_group/main.tf | 6 ++++++ modules/self-managed-node-group/README.md | 1 + modules/self-managed-node-group/main.tf | 10 ++++++++++ modules/self-managed-node-group/variables.tf | 6 ++++++ node_groups.tf | 5 +++-- 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index eb256555d5..bcb4e7cc48 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -97,6 +97,12 @@ module "eks" { self_managed_node_group_defaults = { create_security_group = false + + # enable discovery of autoscaling groups by cluster-autoscaler + autoscaling_group_tags = { + "k8s.io/cluster-autoscaler/enabled" : true, + "k8s.io/cluster-autoscaler/${local.name}" : "owned", + } } self_managed_node_groups = { diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 7c03d9abd7..36bc98eefe 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -77,6 +77,7 @@ module "self_managed_node_group" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance | `string` | `""` | no | +| [autoscaling\_group\_tags](#input\_autoscaling\_group\_tags) | A map of additional tags to add to the autoscaling group created. Tags are applied to the autoscaling group only and are NOT propagated to instances | `map(string)` | `{}` | no | | [availability\_zones](#input\_availability\_zones) | A list of one or more availability zones for the group. Used for EC2-Classic and default subnets when not specified with `subnet_ids` argument. Conflicts with `subnet_ids` | `list(string)` | `null` | no | | [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 3b59e09299..909910ff05 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -400,6 +400,16 @@ resource "aws_autoscaling_group" "this" { } } + dynamic "tag" { + for_each = var.autoscaling_group_tags + + content { + key = tag.key + value = tag.value + propagate_at_launch = false + } + } + timeouts { delete = var.delete_timeout } diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 7351038d54..d3fcd726c2 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -464,6 +464,12 @@ variable "use_default_tags" { default = false } +variable "autoscaling_group_tags" { + description = "A map of additional tags to add to the autoscaling group created. Tags are applied to the autoscaling group only and are NOT propagated to instances" + type = map(string) + default = {} +} + ################################################################################ # Autoscaling group schedule ################################################################################ diff --git a/node_groups.tf b/node_groups.tf index dba8329919..1996befa28 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -387,8 +387,9 @@ module "self_managed_node_group" { create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, false) schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, null) - delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) - use_default_tags = try(each.value.use_default_tags, var.self_managed_node_group_defaults.use_default_tags, false) + delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) + use_default_tags = try(each.value.use_default_tags, var.self_managed_node_group_defaults.use_default_tags, false) + autoscaling_group_tags = try(each.value.autoscaling_group_tags, var.self_managed_node_group_defaults.autoscaling_group_tags, {}) # User data platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, "linux") From a2be69ffb037bb8a6dcf8e8e143416378d07030c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 2 Jun 2022 12:27:21 +0000 Subject: [PATCH 550/962] chore(release): version 18.23.0 [skip ci] ## [18.23.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.22.0...v18.23.0) (2022-06-02) ### Features * Add `autoscaling_group_tags` variable to self-managed-node-groups ([#2084](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2084)) ([8584dcb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8584dcb2e0c9061828505c36a8ed8eb6ced02053)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76d984d8e5..b2dd195347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.23.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.22.0...v18.23.0) (2022-06-02) + + +### Features + +* Add `autoscaling_group_tags` variable to self-managed-node-groups ([#2084](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2084)) ([8584dcb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8584dcb2e0c9061828505c36a8ed8eb6ced02053)) + ## [18.22.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.21.0...v18.22.0) (2022-06-02) From 6bb73bbdd469797cb067e9aa22ce2f31a7fb22be Mon Sep 17 00:00:00 2001 From: Lucas Freitas <4260591+lucasff@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:38:42 +0200 Subject: [PATCH 551/962] docs: Update extraneous markdown tag in `faq.md` (#2092) Co-authored-by: Bryant Biggs --- docs/faq.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 7a52410715..e729a3bf6b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -76,8 +76,6 @@ Note: Windows based node support is limited to a default user data template that Examples of accessing the attributes of the compute resource(s) created by the root module are shown below. Note - the assumption is that your cluster module definition is named `eks` as in `module "eks" { ... }`: -````hcl - - EKS Managed Node Group attributes ```hcl From 140c872eb949081c04f96536c79fabca7b46d78f Mon Sep 17 00:00:00 2001 From: Lucas Freitas <4260591+lucasff@users.noreply.github.com> Date: Sat, 18 Jun 2022 17:02:53 +0200 Subject: [PATCH 552/962] docs: Mention that security groups must be manually referenced on externally attached EKS managed groups (#2101) --- modules/eks-managed-node-group/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 0733ae8642..bec8807abc 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -15,6 +15,11 @@ module "eks_managed_node_group" { vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + // The following variables are necessary if you decide to use the module outside of the parent EKS module context. + // Without it, the security groups of the nodes are empty and thus won't join the cluster. + cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id + cluster_security_group_id = module.eks.node_security_group_id + min_size = 1 max_size = 10 desired_size = 1 From a4c61f41f9483f57931186e9252b3d1dad1d122a Mon Sep 17 00:00:00 2001 From: Basile Chandesris Date: Sat, 18 Jun 2022 17:08:44 +0200 Subject: [PATCH 553/962] docs: Update README to show KMS key ARN not ID (#2098) Co-authored-by: Bryant Biggs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index db840bff74..8a35630602 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ module "eks" { } cluster_encryption_config = [{ - provider_key_arn = "ac01234b-00d9-40f6-ac95-e42345f78b00" + provider_key_arn = "arn:aws:kms:eu-west-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab" resources = ["secrets"] }] From d10b173a2dde0eef7e77deb07c36dcc8439d7aec Mon Sep 17 00:00:00 2001 From: tculp Date: Sat, 18 Jun 2022 11:11:49 -0400 Subject: [PATCH 554/962] Added outputs for labels and taints and updated example to show usage of labels and taints (#2109) Co-authored-by: Bryant Biggs --- examples/eks_managed_node_group/README.md | 1 + examples/eks_managed_node_group/main.tf | 41 +++++++++++++++++++++++ modules/eks-managed-node-group/README.md | 2 ++ modules/eks-managed-node-group/outputs.tf | 10 ++++++ 4 files changed, 54 insertions(+) diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 5293cd5232..8afef350a4 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -80,6 +80,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| +| [aws_autoscaling_group_tag.cluster_autoscaler_label_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group_tag) | resource | | [aws_iam_policy.node_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_key_pair.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 941509afde..13c43c8614 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -668,3 +668,44 @@ data "aws_ami" "eks_default_bottlerocket" { values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"] } } + +################################################################################ +# Tags for the ASG to support cluster-autoscaler scale up from 0 +################################################################################ + +locals { + cluster_autoscaler_label_tags = merge([ + for name, group in module.eks.eks_managed_node_groups : { + for label_name, label_value in coalesce(group.node_group_labels, {}) : "${name}|label|${label_name}" => { + autoscaling_group = group.node_group_autoscaling_group_names[0], + key = "k8s.io/cluster-autoscaler/node-template/label/${label_name}", + value = label_value, + } + } + ]...) + + cluster_autoscaler_taint_tags = merge([ + for name, group in module.eks.eks_managed_node_groups : { + for taint in coalesce(group.node_group_taints, []) : "${name}|taint|${taint.key}" => { + autoscaling_group = group.node_group_autoscaling_group_names[0], + key = "k8s.io/cluster-autoscaler/node-template/taint/${taint.key}" + value = "${taint.value}:${taint.effect}" + } + } + ]...) + + cluster_autoscaler_asg_tags = merge(local.cluster_autoscaler_label_tags, local.cluster_autoscaler_taint_tags) +} + +resource "aws_autoscaling_group_tag" "cluster_autoscaler_label_tags" { + for_each = local.cluster_autoscaler_asg_tags + + autoscaling_group_name = each.value.autoscaling_group + + tag { + key = each.value.key + value = each.value.value + + propagate_at_launch = false + } +} diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index bec8807abc..d1f4272b1b 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -178,8 +178,10 @@ module "eks_managed_node_group" { | [node\_group\_arn](#output\_node\_group\_arn) | Amazon Resource Name (ARN) of the EKS Node Group | | [node\_group\_autoscaling\_group\_names](#output\_node\_group\_autoscaling\_group\_names) | List of the autoscaling group names | | [node\_group\_id](#output\_node\_group\_id) | EKS Cluster name and EKS Node Group name separated by a colon (`:`) | +| [node\_group\_labels](#output\_node\_group\_labels) | Map of labels applied to the node group | | [node\_group\_resources](#output\_node\_group\_resources) | List of objects containing information about underlying resources | | [node\_group\_status](#output\_node\_group\_status) | Status of the EKS Node Group | +| [node\_group\_taints](#output\_node\_group\_taints) | List of objects containing information about taints applied to the node group | | [security\_group\_arn](#output\_security\_group\_arn) | Amazon Resource Name (ARN) of the security group | | [security\_group\_id](#output\_security\_group\_id) | ID of the security group | diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index a73ea1eacf..2899692c6b 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -46,6 +46,16 @@ output "node_group_status" { value = try(aws_eks_node_group.this[0].arn, "") } +output "node_group_labels" { + description = "Map of labels applied to the node group" + value = try(aws_eks_node_group.this[0].labels, {}) +} + +output "node_group_taints" { + description = "List of objects containing information about taints applied to the node group" + value = try(aws_eks_node_group.this[0].taint, []) +} + ################################################################################ # Security Group ################################################################################ From ebc91bcd37a919a350d872a5b235ccc2a79955a6 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 18 Jun 2022 17:52:30 -0400 Subject: [PATCH 555/962] feat: Add support for specifying conrol plane subnets separate from those used by node groups (data plane) (#2113) --- .pre-commit-config.yaml | 4 ++-- README.md | 7 ++++--- docs/compute_resources.md | 6 +++--- docs/irsa_integration.md | 2 +- examples/complete/main.tf | 8 +++++--- examples/eks_managed_node_group/main.tf | 2 +- examples/karpenter/main.tf | 4 ++-- examples/self_managed_node_group/main.tf | 2 +- main.tf | 2 +- modules/eks-managed-node-group/README.md | 2 +- modules/self-managed-node-group/README.md | 2 +- variables.tf | 10 ++++++++-- 12 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0925b0080..27d478d222 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.68.1 + rev: v1.72.1 hooks: - id: terraform_fmt - id: terraform_validate @@ -23,7 +23,7 @@ repos: - '--args=--only=terraform_standard_module_structure' - '--args=--only=terraform_workspace_remote' - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.3.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer diff --git a/README.md b/README.md index 8a35630602..4677dc07b3 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ module "eks" { version = "~> 18.0" cluster_name = "my-cluster" - cluster_version = "1.21" + cluster_version = "1.22" cluster_endpoint_private_access = true cluster_endpoint_public_access = true @@ -292,7 +292,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.21`) | `string` | `null` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.22`) | `string` | `null` | no | +| [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | @@ -327,7 +328,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [putin\_khuylo](#input\_putin\_khuylo) | Do you agree that Putin doesn't respect Ukrainian sovereignty and territorial integrity? More info: https://en.wikipedia.org/wiki/Putin_khuylo! | `bool` | `true` | no | | [self\_managed\_node\_group\_defaults](#input\_self\_managed\_node\_group\_defaults) | Map of self-managed node group default configurations | `any` | `{}` | no | | [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create | `any` | `{}` | no | -| [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the EKS cluster (ENIs) will be provisioned along with the nodes/node groups. Node groups can be deployed within a different set of subnet IDs from within the node group configuration | `list(string)` | `[]` | no | +| [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets | `list(string)` | `[]` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [vpc\_id](#input\_vpc\_id) | ID of the VPC where the cluster and its nodes will be provisioned | `string` | `null` | no | diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 556a2fc84f..2fefd17e9b 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -141,9 +141,9 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 1. The `self-managed-node-group` uses the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version by default: ```hcl - cluster_version = "1.21" + cluster_version = "1.22" - # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.21 + # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.22 self_managed_node_groups = { default = {} } @@ -152,7 +152,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 2. To use Bottlerocket, specify the `platform` as `bottlerocket` and supply a Bottlerocket OS AMI: ```hcl - cluster_version = "1.21" + cluster_version = "1.22" self_managed_node_groups = { bottlerocket = { diff --git a/docs/irsa_integration.md b/docs/irsa_integration.md index 93293e7705..6c78bd9575 100644 --- a/docs/irsa_integration.md +++ b/docs/irsa_integration.md @@ -8,7 +8,7 @@ module "eks" { source = "terraform-aws-modules/eks/aws" cluster_name = "example" - cluster_version = "1.21" + cluster_version = "1.22" cluster_addons = { vpc-cni = { diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 3702dd1b55..69014accb7 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -13,7 +13,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) exec { - api_version = "client.authentication.k8s.io/v1alpha1" + api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] @@ -57,8 +57,9 @@ module "eks" { resources = ["secrets"] }] - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + control_plane_subnet_ids = module.vpc.intra_subnets # Extend cluster security group rules cluster_security_group_additional_rules = { @@ -331,6 +332,7 @@ module "vpc" { azs = ["${local.region}a", "${local.region}b", "${local.region}c"] private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + intra_subnets = ["10.0.7.0/28", "10.0.7.16/28", "10.0.7.32/28"] enable_nat_gateway = true single_nat_gateway = true diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 13c43c8614..63da4c9954 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -7,7 +7,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) exec { - api_version = "client.authentication.k8s.io/v1alpha1" + api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 4cda4c9c3b..3f43d80b2a 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -77,7 +77,7 @@ provider "helm" { cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) exec { - api_version = "client.authentication.k8s.io/v1alpha1" + api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] @@ -92,7 +92,7 @@ provider "kubectl" { load_config_file = false exec { - api_version = "client.authentication.k8s.io/v1alpha1" + api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index bcb4e7cc48..116dbc2553 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -7,7 +7,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) exec { - api_version = "client.authentication.k8s.io/v1alpha1" + api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] diff --git a/main.tf b/main.tf index 9dad2cceee..3254ad1ed1 100644 --- a/main.tf +++ b/main.tf @@ -18,7 +18,7 @@ resource "aws_eks_cluster" "this" { vpc_config { security_group_ids = compact(distinct(concat(var.cluster_additional_security_group_ids, [local.cluster_security_group_id]))) - subnet_ids = var.subnet_ids + subnet_ids = coalescelist(var.control_plane_subnet_ids, var.subnet_ids) endpoint_private_access = var.cluster_endpoint_private_access endpoint_public_access = var.cluster_endpoint_public_access public_access_cidrs = var.cluster_endpoint_public_access_cidrs diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index d1f4272b1b..179090fecd 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -10,7 +10,7 @@ module "eks_managed_node_group" { name = "separate-eks-mng" cluster_name = "my-cluster" - cluster_version = "1.21" + cluster_version = "1.22" vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 36bc98eefe..3cba48859d 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -10,7 +10,7 @@ module "self_managed_node_group" { name = "separate-self-mng" cluster_name = "my-cluster" - cluster_version = "1.21" + cluster_version = "1.22" cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" diff --git a/variables.tf b/variables.tf index 2890834184..094711f109 100644 --- a/variables.tf +++ b/variables.tf @@ -27,7 +27,7 @@ variable "cluster_name" { } variable "cluster_version" { - description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.21`)" + description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.22`)" type = string default = null } @@ -44,8 +44,14 @@ variable "cluster_additional_security_group_ids" { default = [] } +variable "control_plane_subnet_ids" { + description = "A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane" + type = list(string) + default = [] +} + variable "subnet_ids" { - description = "A list of subnet IDs where the EKS cluster (ENIs) will be provisioned along with the nodes/node groups. Node groups can be deployed within a different set of subnet IDs from within the node group configuration" + description = "A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets" type = list(string) default = [] } From ad6b2e2ab77d35d86028d69fea886f7148db7bb8 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 18 Jun 2022 21:53:07 +0000 Subject: [PATCH 556/962] chore(release): version 18.24.0 [skip ci] ## [18.24.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.23.0...v18.24.0) (2022-06-18) ### Features * Add support for specifying conrol plane subnets separate from those used by node groups (data plane) ([#2113](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2113)) ([ebc91bc](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ebc91bcd37a919a350d872a5b235ccc2a79955a6)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2dd195347..78541efb5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.24.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.23.0...v18.24.0) (2022-06-18) + + +### Features + +* Add support for specifying conrol plane subnets separate from those used by node groups (data plane) ([#2113](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2113)) ([ebc91bc](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ebc91bcd37a919a350d872a5b235ccc2a79955a6)) + ## [18.23.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.22.0...v18.23.0) (2022-06-02) From 5a5a32ed1241ba3cc64abe37b37bcb5ad52d42c4 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 19 Jun 2022 08:02:07 -0400 Subject: [PATCH 557/962] fix: Remove `modified_at` from ignored changes on EKS addons (#2114) --- main.tf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/main.tf b/main.tf index 3254ad1ed1..463d3b677d 100644 --- a/main.tf +++ b/main.tf @@ -312,12 +312,6 @@ resource "aws_eks_addon" "this" { resolve_conflicts = lookup(each.value, "resolve_conflicts", null) service_account_role_arn = lookup(each.value, "service_account_role_arn", null) - lifecycle { - ignore_changes = [ - modified_at - ] - } - depends_on = [ module.fargate_profile, module.eks_managed_node_group, From 85e5cae702bafcb81e7e74ec1497cda772de4b86 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 19 Jun 2022 12:02:48 +0000 Subject: [PATCH 558/962] chore(release): version 18.24.1 [skip ci] ### [18.24.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.24.0...v18.24.1) (2022-06-19) ### Bug Fixes * Remove `modified_at` from ignored changes on EKS addons ([#2114](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2114)) ([5a5a32e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5a5a32ed1241ba3cc64abe37b37bcb5ad52d42c4)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78541efb5e..aa4c564468 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.24.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.24.0...v18.24.1) (2022-06-19) + + +### Bug Fixes + +* Remove `modified_at` from ignored changes on EKS addons ([#2114](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2114)) ([5a5a32e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5a5a32ed1241ba3cc64abe37b37bcb5ad52d42c4)) + ## [18.24.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.23.0...v18.24.0) (2022-06-18) From e0f5995a8b08b89cc6e805a9874d8265230b6a93 Mon Sep 17 00:00:00 2001 From: wreulicke <12907474+wreulicke@users.noreply.github.com> Date: Sun, 26 Jun 2022 23:15:34 +0900 Subject: [PATCH 559/962] docs: Update link for EKS service role (#2118) --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 463d3b677d..4f3a36e467 100644 --- a/main.tf +++ b/main.tf @@ -252,7 +252,7 @@ resource "aws_iam_role" "this" { tags = merge(var.tags, var.iam_role_tags) } -# Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group +# Policies attached ref https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html resource "aws_iam_role_policy_attachment" "this" { for_each = local.create_iam_role ? toset(compact(distinct(concat([ "${local.policy_arn_prefix}/AmazonEKSClusterPolicy", From 75acb09ec56c5ce8e5f74ebc7bf15468b272db8a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 28 Jun 2022 10:55:39 -0400 Subject: [PATCH 560/962] feat: Add support for creating KMS key for cluster secret encryption (#2121) * feat: Add support for creating KMS key for cluster secret encryption * fix: Update naming scheme to include `kms_` --- .pre-commit-config.yaml | 2 +- README.md | 19 ++++- examples/complete/README.md | 4 +- examples/complete/main.tf | 15 ++-- examples/complete/outputs.tf | 19 +++++ examples/eks_managed_node_group/README.md | 3 + examples/eks_managed_node_group/outputs.tf | 19 +++++ examples/fargate_profile/README.md | 3 + examples/fargate_profile/outputs.tf | 19 +++++ examples/self_managed_node_group/README.md | 3 + examples/self_managed_node_group/outputs.tf | 19 +++++ main.tf | 39 +++++++++- outputs.tf | 19 +++++ variables.tf | 83 +++++++++++++++++++-- 14 files changed, 245 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27d478d222..9eaabc7a07 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.72.1 + rev: v1.73.0 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/README.md b/README.md index 4677dc07b3..5931adafd2 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|--------|---------| | [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ./modules/eks-managed-node-group | n/a | | [fargate\_profile](#module\_fargate\_profile) | ./modules/fargate-profile | n/a | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 1.0.0 | | [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ./modules/self-managed-node-group | n/a | ## Resources @@ -249,6 +250,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | | [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -270,7 +272,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | | [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no | | [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | -| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster |
list(object({
provider_key_arn = string
resources = list(string)
}))
| `[]` | no | +| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster | `list(any)` | `[]` | no | | [cluster\_encryption\_policy\_description](#input\_cluster\_encryption\_policy\_description) | Description of the cluster encryption policy created | `string` | `"Cluster encryption policy to allow cluster role to utilize CMK provided"` | no | | [cluster\_encryption\_policy\_name](#input\_cluster\_encryption\_policy\_name) | Name to use on cluster encryption policy created | `string` | `null` | no | | [cluster\_encryption\_policy\_path](#input\_cluster\_encryption\_policy\_path) | Cluster encryption policy path | `string` | `null` | no | @@ -301,11 +303,13 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster or use the existing `cluster_security_group_id` | `bool` | `true` | no | | [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy) | `bool` | `false` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether a an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_kms\_key](#input\_create\_kms\_key) | Controls if a KMS key for cluster encryption should be created | `bool` | `false` | no | | [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | | [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | +| [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled. Defaults to `true` | `bool` | `true` | no | | [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no | | [fargate\_profiles](#input\_fargate\_profiles) | Map of Fargate Profile definitions to create | `any` | `{}` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | @@ -316,6 +320,16 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [kms\_key\_administrators](#input\_kms\_key\_administrators) | A list of IAM ARNs for [key administrators](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-administrators). If no value is provided, the current caller identity is used to ensure at least one key admin is available | `list(string)` | `[]` | no | +| [kms\_key\_aliases](#input\_kms\_key\_aliases) | A list of aliases to create. Note - due to the use of `toset()`, values must be static strings and not computed values | `list(string)` | `[]` | no | +| [kms\_key\_deletion\_window\_in\_days](#input\_kms\_key\_deletion\_window\_in\_days) | The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key. If you specify a value, it must be between `7` and `30`, inclusive. If you do not specify a value, it defaults to `30` | `number` | `null` | no | +| [kms\_key\_description](#input\_kms\_key\_description) | The description of the key as viewed in AWS console | `string` | `null` | no | +| [kms\_key\_enable\_default\_policy](#input\_kms\_key\_enable\_default\_policy) | Specifies whether to enable the default key policy. Defaults to `true` | `bool` | `false` | no | +| [kms\_key\_override\_policy\_documents](#input\_kms\_key\_override\_policy\_documents) | List of IAM policy documents that are merged together into the exported document. In merging, statements with non-blank `sid`s will override statements with the same `sid` | `list(string)` | `[]` | no | +| [kms\_key\_owners](#input\_kms\_key\_owners) | A list of IAM ARNs for those who will have full key permissions (`kms:*`) | `list(string)` | `[]` | no | +| [kms\_key\_service\_users](#input\_kms\_key\_service\_users) | A list of IAM ARNs for [key service users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-service-integration) | `list(string)` | `[]` | no | +| [kms\_key\_source\_policy\_documents](#input\_kms\_key\_source\_policy\_documents) | List of IAM policy documents that are merged together into the exported document. Statements must have unique `sid`s | `list(string)` | `[]` | no | +| [kms\_key\_users](#input\_kms\_key\_users) | A list of IAM ARNs for [key users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-users) | `list(string)` | `[]` | no | | [manage\_aws\_auth\_configmap](#input\_manage\_aws\_auth\_configmap) | Determines whether to manage the aws-auth configmap | `bool` | `false` | no | | [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no | | [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | @@ -358,6 +372,9 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | +| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | +| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/examples/complete/README.md b/examples/complete/README.md index d329fad1d0..7aa52f0740 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -61,7 +61,6 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| -| [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | ## Inputs @@ -92,6 +91,9 @@ No inputs. | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | +| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | +| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 69014accb7..91458939b5 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -52,10 +52,13 @@ module "eks" { } } + # Encryption key + create_kms_key = true cluster_encryption_config = [{ - provider_key_arn = aws_kms_key.eks.arn - resources = ["secrets"] + resources = ["secrets"] }] + kms_key_deletion_window_in_days = 7 + enable_kms_key_rotation = true vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets @@ -372,11 +375,3 @@ resource "aws_security_group" "additional" { tags = local.tags } - -resource "aws_kms_key" "eks" { - description = "EKS Secret Encryption Key" - deletion_window_in_days = 7 - enable_key_rotation = true - - tags = local.tags -} diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index cbfae9a452..c612b0fe3d 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -42,6 +42,25 @@ output "cluster_security_group_id" { value = module.eks.cluster_security_group_id } +################################################################################ +# KMS Key +################################################################################ + +output "kms_key_arn" { + description = "The Amazon Resource Name (ARN) of the key" + value = module.eks.kms_key_arn +} + +output "kms_key_id" { + description = "The globally unique identifier for the key" + value = module.eks.kms_key_id +} + +output "kms_key_policy" { + description = "The IAM resource policy set on the key" + value = module.eks.kms_key_policy +} + ################################################################################ # Security Group ################################################################################ diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 8afef350a4..71eb4fa26d 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -125,6 +125,9 @@ No inputs. | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | +| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | +| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/examples/eks_managed_node_group/outputs.tf b/examples/eks_managed_node_group/outputs.tf index bfe53985b1..3e9e8dd23e 100644 --- a/examples/eks_managed_node_group/outputs.tf +++ b/examples/eks_managed_node_group/outputs.tf @@ -42,6 +42,25 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +################################################################################ +# KMS Key +################################################################################ + +output "kms_key_arn" { + description = "The Amazon Resource Name (ARN) of the key" + value = module.eks.kms_key_arn +} + +output "kms_key_id" { + description = "The globally unique identifier for the key" + value = module.eks.kms_key_id +} + +output "kms_key_policy" { + description = "The IAM resource policy set on the key" + value = module.eks.kms_key_policy +} + ################################################################################ # Security Group ################################################################################ diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 1dcb8638c8..81f0366692 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -71,6 +71,9 @@ No inputs. | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | +| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | +| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/examples/fargate_profile/outputs.tf b/examples/fargate_profile/outputs.tf index bfe53985b1..3e9e8dd23e 100644 --- a/examples/fargate_profile/outputs.tf +++ b/examples/fargate_profile/outputs.tf @@ -42,6 +42,25 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +################################################################################ +# KMS Key +################################################################################ + +output "kms_key_arn" { + description = "The Amazon Resource Name (ARN) of the key" + value = module.eks.kms_key_arn +} + +output "kms_key_id" { + description = "The globally unique identifier for the key" + value = module.eks.kms_key_id +} + +output "kms_key_policy" { + description = "The IAM resource policy set on the key" + value = module.eks.kms_key_policy +} + ################################################################################ # Security Group ################################################################################ diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index d668611bd7..a0caf4a2e2 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -88,6 +88,9 @@ No inputs. | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | +| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | +| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/examples/self_managed_node_group/outputs.tf b/examples/self_managed_node_group/outputs.tf index bfe53985b1..3e9e8dd23e 100644 --- a/examples/self_managed_node_group/outputs.tf +++ b/examples/self_managed_node_group/outputs.tf @@ -42,6 +42,25 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +################################################################################ +# KMS Key +################################################################################ + +output "kms_key_arn" { + description = "The Amazon Resource Name (ARN) of the key" + value = module.eks.kms_key_arn +} + +output "kms_key_id" { + description = "The globally unique identifier for the key" + value = module.eks.kms_key_id +} + +output "kms_key_policy" { + description = "The IAM resource policy set on the key" + value = module.eks.kms_key_policy +} + ################################################################################ # Security Group ################################################################################ diff --git a/main.tf b/main.tf index 4f3a36e467..916a62e635 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,10 @@ data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} locals { create = var.create && var.putin_khuylo + + cluster_role = try(aws_iam_role.this[0].arn, var.iam_role_arn) } ################################################################################ @@ -12,7 +15,7 @@ resource "aws_eks_cluster" "this" { count = local.create ? 1 : 0 name = var.cluster_name - role_arn = try(aws_iam_role.this[0].arn, var.iam_role_arn) + role_arn = local.cluster_role version = var.cluster_version enabled_cluster_log_types = var.cluster_enabled_log_types @@ -34,7 +37,7 @@ resource "aws_eks_cluster" "this" { content { provider { - key_arn = encryption_config.value.provider_key_arn + key_arn = var.create_kms_key ? module.kms.key_arn : encryption_config.value.provider_key_arn } resources = encryption_config.value.resources } @@ -80,6 +83,36 @@ resource "aws_cloudwatch_log_group" "this" { tags = var.tags } +################################################################################ +# KMS Key +################################################################################ + +module "kms" { + source = "terraform-aws-modules/kms/aws" + version = "1.0.0" + + create = var.create_kms_key + + description = coalesce(var.kms_key_description, "${var.cluster_name} cluster encryption key") + key_usage = "ENCRYPT_DECRYPT" + deletion_window_in_days = var.kms_key_deletion_window_in_days + enable_key_rotation = var.enable_kms_key_rotation + + # Policy + enable_default_policy = var.kms_key_enable_default_policy + key_owners = var.kms_key_owners + key_administrators = coalescelist(var.kms_key_administrators, [data.aws_caller_identity.current.arn]) + key_users = concat([local.cluster_role], var.kms_key_users) + key_service_users = var.kms_key_service_users + source_policy_documents = var.kms_key_source_policy_documents + override_policy_documents = var.kms_key_override_policy_documents + + # Aliases + aliases = concat(["eks/${var.cluster_name}"], var.kms_key_aliases) + + tags = var.tags +} + ################################################################################ # Cluster Security Group # Defaults follow https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html @@ -290,7 +323,7 @@ resource "aws_iam_policy" "cluster_encryption" { "kms:DescribeKey", ] Effect = "Allow" - Resource = [for config in var.cluster_encryption_config : config.provider_key_arn] + Resource = var.create_kms_key ? [module.kms.key_arn] : [for config in var.cluster_encryption_config : config.provider_key_arn] }, ] }) diff --git a/outputs.tf b/outputs.tf index 1245e43777..8ff30fc623 100644 --- a/outputs.tf +++ b/outputs.tf @@ -47,6 +47,25 @@ output "cluster_primary_security_group_id" { value = try(aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id, "") } +################################################################################ +# KMS Key +################################################################################ + +output "kms_key_arn" { + description = "The Amazon Resource Name (ARN) of the key" + value = module.kms.key_arn +} + +output "kms_key_id" { + description = "The globally unique identifier for the key" + value = module.kms.key_id +} + +output "kms_key_policy" { + description = "The IAM resource policy set on the key" + value = module.kms.key_policy +} + ################################################################################ # Cluster Security Group ################################################################################ diff --git a/variables.tf b/variables.tf index 094711f109..0f2c2df99b 100644 --- a/variables.tf +++ b/variables.tf @@ -88,11 +88,8 @@ variable "cluster_service_ipv4_cidr" { variable "cluster_encryption_config" { description = "Configuration block with encryption configuration for the cluster" - type = list(object({ - provider_key_arn = string - resources = list(string) - })) - default = [] + type = list(any) + default = [] } variable "attach_cluster_encryption_policy" { @@ -119,6 +116,82 @@ variable "cluster_timeouts" { default = {} } +################################################################################ +# KMS Key +################################################################################ + +variable "create_kms_key" { + description = "Controls if a KMS key for cluster encryption should be created" + type = bool + default = false +} + +variable "kms_key_description" { + description = "The description of the key as viewed in AWS console" + type = string + default = null +} + +variable "kms_key_deletion_window_in_days" { + description = "The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key. If you specify a value, it must be between `7` and `30`, inclusive. If you do not specify a value, it defaults to `30`" + type = number + default = null +} + +variable "enable_kms_key_rotation" { + description = "Specifies whether key rotation is enabled. Defaults to `true`" + type = bool + default = true +} + +variable "kms_key_enable_default_policy" { + description = "Specifies whether to enable the default key policy. Defaults to `true`" + type = bool + default = false +} + +variable "kms_key_owners" { + description = "A list of IAM ARNs for those who will have full key permissions (`kms:*`)" + type = list(string) + default = [] +} + +variable "kms_key_administrators" { + description = "A list of IAM ARNs for [key administrators](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-administrators). If no value is provided, the current caller identity is used to ensure at least one key admin is available" + type = list(string) + default = [] +} + +variable "kms_key_users" { + description = "A list of IAM ARNs for [key users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-users)" + type = list(string) + default = [] +} + +variable "kms_key_service_users" { + description = "A list of IAM ARNs for [key service users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-service-integration)" + type = list(string) + default = [] +} + +variable "kms_key_source_policy_documents" { + description = "List of IAM policy documents that are merged together into the exported document. Statements must have unique `sid`s" + type = list(string) + default = [] +} + +variable "kms_key_override_policy_documents" { + description = "List of IAM policy documents that are merged together into the exported document. In merging, statements with non-blank `sid`s will override statements with the same `sid`" + type = list(string) + default = [] +} + +variable "kms_key_aliases" { + description = "A list of aliases to create. Note - due to the use of `toset()`, values must be static strings and not computed values" + type = list(string) + default = [] +} + ################################################################################ # CloudWatch Log Group ################################################################################ From 62b776f142f6e3e4dd3ef927c8fef12b23401851 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 28 Jun 2022 14:56:13 +0000 Subject: [PATCH 561/962] chore(release): version 18.25.0 [skip ci] ## [18.25.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.24.1...v18.25.0) (2022-06-28) ### Features * Add support for creating KMS key for cluster secret encryption ([#2121](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2121)) ([75acb09](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/75acb09ec56c5ce8e5f74ebc7bf15468b272db8a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa4c564468..9421741e2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.25.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.24.1...v18.25.0) (2022-06-28) + + +### Features + +* Add support for creating KMS key for cluster secret encryption ([#2121](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2121)) ([75acb09](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/75acb09ec56c5ce8e5f74ebc7bf15468b272db8a)) + ### [18.24.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.24.0...v18.24.1) (2022-06-19) From 4543ab454bea80b64381b88a631d955a7cfae247 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 28 Jun 2022 12:16:20 -0400 Subject: [PATCH 562/962] feat: Add support for specifiying NTP address to use private Amazon Time Sync Service (#2125) --- README.md | 2 ++ examples/complete/main.tf | 1 + examples/eks_managed_node_group/main.tf | 1 + node_groups.tf | 8 ++++---- variables.tf | 14 ++++++++++++++ 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5931adafd2..c7fca3e552 100644 --- a/README.md +++ b/README.md @@ -335,6 +335,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | | [node\_security\_group\_id](#input\_node\_security\_group\_id) | ID of an existing security group to attach to the node groups created | `string` | `""` | no | | [node\_security\_group\_name](#input\_node\_security\_group\_name) | Name to use on node security group created | `string` | `null` | no | +| [node\_security\_group\_ntp\_ipv4\_cidr\_block](#input\_node\_security\_group\_ntp\_ipv4\_cidr\_block) | IPv4 CIDR block to allow NTP egress. Default is public IP space, but [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) can be used as well with `["169.254.169.123/32"]` | `list(string)` |
[
"0.0.0.0/0"
]
| no | +| [node\_security\_group\_ntp\_ipv6\_cidr\_block](#input\_node\_security\_group\_ntp\_ipv6\_cidr\_block) | IPv4 CIDR block to allow NTP egress. Default is public IP space, but [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) can be used as well with `["fd00:ec2::123/128"]` | `list(string)` |
[
"::/0"
]
| no | | [node\_security\_group\_tags](#input\_node\_security\_group\_tags) | A map of additional tags to add to the node security group created | `map(string)` | `{}` | no | | [node\_security\_group\_use\_name\_prefix](#input\_node\_security\_group\_use\_name\_prefix) | Determines whether node security group name (`node_security_group_name`) is used as a prefix | `string` | `true` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider | `list(string)` | `[]` | no | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 91458939b5..55fc8bf38c 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -77,6 +77,7 @@ module "eks" { } # Extend node-to-node security group rules + node_security_group_ntp_ipv4_cidr_block = ["169.254.169.123/32"] node_security_group_additional_rules = { ingress_self_all = { description = "Node to node all ports/protocols" diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 63da4c9954..b8f7fd60ab 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -92,6 +92,7 @@ module "eks" { } # Extend node-to-node security group rules + node_security_group_ntp_ipv4_cidr_block = ["fd00:ec2::123/128"] node_security_group_additional_rules = { ingress_self_all = { description = "Node to node all ports/protocols" diff --git a/node_groups.tf b/node_groups.tf index 1996befa28..8c0916ade3 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -130,8 +130,8 @@ locals { from_port = 123 to_port = 123 type = "egress" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null + cidr_blocks = var.node_security_group_ntp_ipv4_cidr_block + ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? var.node_security_group_ntp_ipv6_cidr_block : null } egress_ntp_udp = { description = "Egress NTP/UDP to internet" @@ -139,8 +139,8 @@ locals { from_port = 123 to_port = 123 type = "egress" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null + cidr_blocks = var.node_security_group_ntp_ipv4_cidr_block + ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? var.node_security_group_ntp_ipv6_cidr_block : null } } } diff --git a/variables.tf b/variables.tf index 0f2c2df99b..ad7777063f 100644 --- a/variables.tf +++ b/variables.tf @@ -322,6 +322,20 @@ variable "node_security_group_tags" { default = {} } +# TODO - at next breaking change, make 169.254.169.123/32 the default +variable "node_security_group_ntp_ipv4_cidr_block" { + description = "IPv4 CIDR block to allow NTP egress. Default is public IP space, but [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) can be used as well with `[\"169.254.169.123/32\"]`" + type = list(string) + default = ["0.0.0.0/0"] +} + +# TODO - at next breaking change, make fd00:ec2::123/128 the default +variable "node_security_group_ntp_ipv6_cidr_block" { + description = "IPv4 CIDR block to allow NTP egress. Default is public IP space, but [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) can be used as well with `[\"fd00:ec2::123/128\"]`" + type = list(string) + default = ["::/0"] +} + ################################################################################ # IRSA ################################################################################ From 9eeb3e3270b400bef5b2713793fb7e63432a5b84 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 28 Jun 2022 16:17:04 +0000 Subject: [PATCH 563/962] chore(release): version 18.26.0 [skip ci] ## [18.26.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.25.0...v18.26.0) (2022-06-28) ### Features * Add support for specifiying NTP address to use private Amazon Time Sync Service ([#2125](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2125)) ([4543ab4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/4543ab454bea80b64381b88a631d955a7cfae247)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9421741e2f..4ce4a59614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.26.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.25.0...v18.26.0) (2022-06-28) + + +### Features + +* Add support for specifiying NTP address to use private Amazon Time Sync Service ([#2125](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2125)) ([4543ab4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/4543ab454bea80b64381b88a631d955a7cfae247)) + ## [18.25.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.24.1...v18.25.0) (2022-06-28) From bc04cd3a0a4286566ea56b20d9314115c6e489ab Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 29 Jun 2022 07:51:44 -0400 Subject: [PATCH 564/962] fix: Update KMS module version which aligns on module version requirements (#2127) --- README.md | 2 +- main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c7fca3e552..4c90720368 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|--------|---------| | [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ./modules/eks-managed-node-group | n/a | | [fargate\_profile](#module\_fargate\_profile) | ./modules/fargate-profile | n/a | -| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 1.0.0 | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 1.0.1 | | [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ./modules/self-managed-node-group | n/a | ## Resources diff --git a/main.tf b/main.tf index 916a62e635..0924a6e8f3 100644 --- a/main.tf +++ b/main.tf @@ -89,7 +89,7 @@ resource "aws_cloudwatch_log_group" "this" { module "kms" { source = "terraform-aws-modules/kms/aws" - version = "1.0.0" + version = "1.0.1" # Note - be mindful of Terraform/provider version compatibility between modules create = var.create_kms_key From f34670cf0bc84c42df14f64422ffda7298006fb8 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 29 Jun 2022 11:52:26 +0000 Subject: [PATCH 565/962] chore(release): version 18.26.1 [skip ci] ### [18.26.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.0...v18.26.1) (2022-06-29) ### Bug Fixes * Update KMS module version which aligns on module version requirements ([#2127](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2127)) ([bc04cd3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/bc04cd3a0a4286566ea56b20d9314115c6e489ab)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ce4a59614..a38ae86a05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.26.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.0...v18.26.1) (2022-06-29) + + +### Bug Fixes + +* Update KMS module version which aligns on module version requirements ([#2127](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2127)) ([bc04cd3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/bc04cd3a0a4286566ea56b20d9314115c6e489ab)) + ## [18.26.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.25.0...v18.26.0) (2022-06-28) From 2d7701c3b0f2c6dcc10f31fc1f703bfde31b2c5b Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 1 Jul 2022 07:53:43 -0400 Subject: [PATCH 566/962] fix: Correct variable types to improve dynamic check correctness (#2133) --- modules/eks-managed-node-group/README.md | 18 +++---- modules/eks-managed-node-group/main.tf | 22 ++++---- modules/eks-managed-node-group/variables.tf | 18 +++---- modules/self-managed-node-group/README.md | 30 +++++------ modules/self-managed-node-group/main.tf | 32 +++++------ modules/self-managed-node-group/variables.tf | 30 +++++------ node_groups.tf | 56 ++++++++++---------- 7 files changed, 103 insertions(+), 103 deletions(-) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 179090fecd..4ec3c782c9 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -91,7 +91,7 @@ module "eks_managed_node_group" { | [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Valid values are `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `CUSTOM`, `BOTTLEROCKET_ARM_64`, `BOTTLEROCKET_x86_64` | `string` | `null` | no | | [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | -| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `null` | no | +| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | | [capacity\_type](#input\_capacity\_type) | Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT` | `string` | `"ON_DEMAND"` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | @@ -101,21 +101,21 @@ module "eks_managed_node_group" { | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes version. Defaults to EKS Cluster Kubernetes version | `string` | `null` | no | -| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `null` | no | +| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | | [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | | [create\_security\_group](#input\_create\_security\_group) | Determines whether to create a security group | `bool` | `true` | no | -| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `null` | no | +| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | | [desired\_size](#input\_desired\_size) | Desired number of instances/nodes | `number` | `1` | no | | [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | | [disk\_size](#input\_disk\_size) | Disk size in GiB for nodes. Defaults to `20` | `number` | `null` | no | | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance(s) will be EBS-optimized | `bool` | `null` | no | -| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `map(string)` | `null` | no | -| [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `null` | no | +| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `map(string)` | `{}` | no | +| [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | | [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template | `bool` | `false` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | -| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `null` | no | +| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | | [force\_update\_version](#input\_force\_update\_version) | Force version update if existing pods are unable to be drained due to a pod disruption budget issue | `bool` | `null` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the node group. Required if `create_iam_role` is set to `false` | `string` | `null` | no | @@ -126,7 +126,7 @@ module "eks_managed_node_group" { | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | -| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `null` | no | +| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `{}` | no | | [instance\_types](#input\_instance\_types) | Set of instance types associated with the EKS Node Group. Defaults to `["t3.medium"]` | `list(string)` | `null` | no | | [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | | [key\_name](#input\_key\_name) | The key name that should be used for the instance(s) | `string` | `null` | no | @@ -137,13 +137,13 @@ module "eks_managed_node_group" { | [launch\_template\_tags](#input\_launch\_template\_tags) | A map of additional tags to add to the tag\_specifications of launch template created | `map(string)` | `{}` | no | | [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | | [launch\_template\_version](#input\_launch\_template\_version) | Launch template version number. The default is `$Default` | `string` | `null` | no | -| [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with | `map(string)` | `null` | no | +| [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with | `map(string)` | `{}` | no | | [max\_size](#input\_max\_size) | Maximum number of instances/nodes | `number` | `3` | no | | [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | | [min\_size](#input\_min\_size) | Minimum number of instances/nodes | `number` | `0` | no | | [name](#input\_name) | Name of the EKS managed node group | `string` | `""` | no | | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | -| [placement](#input\_placement) | The placement of the instance | `map(string)` | `null` | no | +| [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | | [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket` or `linux` based; `windows` is not supported | `string` | `"linux"` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | | [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index b18335de20..f55630876e 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -91,7 +91,7 @@ resource "aws_launch_template" "this" { } dynamic "capacity_reservation_specification" { - for_each = var.capacity_reservation_specification != null ? [var.capacity_reservation_specification] : [] + for_each = length(var.capacity_reservation_specification) > 0 ? [var.capacity_reservation_specification] : [] content { capacity_reservation_preference = lookup(capacity_reservation_specification.value, "capacity_reservation_preference", null) @@ -105,7 +105,7 @@ resource "aws_launch_template" "this" { } dynamic "cpu_options" { - for_each = var.cpu_options != null ? [var.cpu_options] : [] + for_each = length(var.cpu_options) > 0 ? [var.cpu_options] : [] content { core_count = cpu_options.value.core_count threads_per_core = cpu_options.value.threads_per_core @@ -113,28 +113,28 @@ resource "aws_launch_template" "this" { } dynamic "credit_specification" { - for_each = var.credit_specification != null ? [var.credit_specification] : [] + for_each = length(var.credit_specification) > 0 ? [var.credit_specification] : [] content { cpu_credits = credit_specification.value.cpu_credits } } dynamic "elastic_gpu_specifications" { - for_each = var.elastic_gpu_specifications != null ? [var.elastic_gpu_specifications] : [] + for_each = length(var.elastic_gpu_specifications) > 0 ? [var.elastic_gpu_specifications] : [] content { type = elastic_gpu_specifications.value.type } } dynamic "elastic_inference_accelerator" { - for_each = var.elastic_inference_accelerator != null ? [var.elastic_inference_accelerator] : [] + for_each = length(var.elastic_inference_accelerator) > 0 ? [var.elastic_inference_accelerator] : [] content { type = elastic_inference_accelerator.value.type } } dynamic "enclave_options" { - for_each = var.enclave_options != null ? [var.enclave_options] : [] + for_each = length(var.enclave_options) > 0 ? [var.enclave_options] : [] content { enabled = enclave_options.value.enabled } @@ -160,12 +160,12 @@ resource "aws_launch_template" "this" { # } dynamic "instance_market_options" { - for_each = var.instance_market_options != null ? [var.instance_market_options] : [] + for_each = length(var.instance_market_options) > 0 ? [var.instance_market_options] : [] content { market_type = instance_market_options.value.market_type dynamic "spot_options" { - for_each = lookup(instance_market_options.value, "spot_options", null) != null ? [instance_market_options.value.spot_options] : [] + for_each = length(lookup(instance_market_options.value, "spot_options", {})) > 0 ? [instance_market_options.value.spot_options] : [] content { block_duration_minutes = lookup(spot_options.value, "block_duration_minutes", null) instance_interruption_behavior = lookup(spot_options.value, "instance_interruption_behavior", null) @@ -178,14 +178,14 @@ resource "aws_launch_template" "this" { } dynamic "license_specification" { - for_each = var.license_specifications != null ? [var.license_specifications] : [] + for_each = length(var.license_specifications) > 0 ? [var.license_specifications] : [] content { license_configuration_arn = license_specifications.value.license_configuration_arn } } dynamic "metadata_options" { - for_each = var.metadata_options != null ? [var.metadata_options] : [] + for_each = length(var.metadata_options) > 0 ? [var.metadata_options] : [] content { http_endpoint = lookup(metadata_options.value, "http_endpoint", null) http_tokens = lookup(metadata_options.value, "http_tokens", null) @@ -225,7 +225,7 @@ resource "aws_launch_template" "this" { } dynamic "placement" { - for_each = var.placement != null ? [var.placement] : [] + for_each = length(var.placement) > 0 ? [var.placement] : [] content { affinity = lookup(placement.value, "affinity", null) availability_zone = lookup(placement.value, "availability_zone", null) diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 5929817887..2398fb1d7a 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -171,49 +171,49 @@ variable "block_device_mappings" { variable "capacity_reservation_specification" { description = "Targeting for EC2 capacity reservations" type = any - default = null + default = {} } variable "cpu_options" { description = "The CPU options for the instance" type = map(string) - default = null + default = {} } variable "credit_specification" { description = "Customize the credit specification of the instance" type = map(string) - default = null + default = {} } variable "elastic_gpu_specifications" { description = "The elastic GPU to attach to the instance" type = map(string) - default = null + default = {} } variable "elastic_inference_accelerator" { description = "Configuration block containing an Elastic Inference Accelerator to attach to the instance" type = map(string) - default = null + default = {} } variable "enclave_options" { description = "Enable Nitro Enclaves on launched instances" type = map(string) - default = null + default = {} } variable "instance_market_options" { description = "The market (purchasing) option for the instance" type = any - default = null + default = {} } variable "license_specifications" { description = "A list of license specifications to associate with" type = map(string) - default = null + default = {} } variable "metadata_options" { @@ -241,7 +241,7 @@ variable "network_interfaces" { variable "placement" { description = "The placement of the instance" type = map(string) - default = null + default = {} } variable "launch_template_tags" { diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 3cba48859d..43b5eb4ba6 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -82,7 +82,7 @@ module "self_managed_node_group" { | [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [capacity\_rebalance](#input\_capacity\_rebalance) | Indicates whether capacity rebalance is enabled | `bool` | `null` | no | -| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `null` | no | +| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | @@ -90,28 +90,28 @@ module "self_managed_node_group" { | [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | -| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `null` | no | +| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | | [create](#input\_create) | Determines whether to create self managed node group or not | `bool` | `true` | no | | [create\_autoscaling\_group](#input\_create\_autoscaling\_group) | Determines whether to create autoscaling group or not | `bool` | `true` | no | | [create\_iam\_instance\_profile](#input\_create\_iam\_instance\_profile) | Determines whether an IAM instance profile is created or to use an existing IAM instance profile | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create launch template or not | `bool` | `true` | no | | [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | | [create\_security\_group](#input\_create\_security\_group) | Determines whether to create a security group | `bool` | `true` | no | -| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `null` | no | +| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | | [default\_cooldown](#input\_default\_cooldown) | The amount of time, in seconds, after a scaling activity completes before another scaling activity can start | `number` | `null` | no | | [delete\_timeout](#input\_delete\_timeout) | Delete timeout to wait for destroying autoscaling group | `string` | `null` | no | | [desired\_size](#input\_desired\_size) | The number of Amazon EC2 instances that should be running in the autoscaling group | `number` | `1` | no | | [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance will be EBS-optimized | `bool` | `null` | no | -| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `map(string)` | `null` | no | -| [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `null` | no | +| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `map(string)` | `{}` | no | +| [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | -| [enabled\_metrics](#input\_enabled\_metrics) | A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances` | `list(string)` | `null` | no | -| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `null` | no | +| [enabled\_metrics](#input\_enabled\_metrics) | A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances` | `list(string)` | `[]` | no | +| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | | [force\_delete](#input\_force\_delete) | Allows deleting the Auto Scaling Group without waiting for all instances in the pool to terminate. You can force an Auto Scaling Group to delete even if it's in the process of scaling a resource. Normally, Terraform drains all the instances before deleting the group. This bypasses that behavior and potentially leaves resources dangling | `bool` | `null` | no | | [health\_check\_grace\_period](#input\_health\_check\_grace\_period) | Time (in seconds) after instance comes into service before checking health | `number` | `null` | no | | [health\_check\_type](#input\_health\_check\_type) | `EC2` or `ELB`. Controls how health checking is done | `string` | `null` | no | -| [hibernation\_options](#input\_hibernation\_options) | The hibernation options for the instance | `map(string)` | `null` | no | +| [hibernation\_options](#input\_hibernation\_options) | The hibernation options for the instance | `map(string)` | `{}` | no | | [iam\_instance\_profile\_arn](#input\_iam\_instance\_profile\_arn) | Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false` | `string` | `null` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | | [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | @@ -123,8 +123,8 @@ module "self_managed_node_group" { | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [initial\_lifecycle\_hooks](#input\_initial\_lifecycle\_hooks) | One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource | `list(map(string))` | `[]` | no | | [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | -| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `null` | no | -| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated | `any` | `null` | no | +| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `{}` | no | +| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated | `any` | `{}` | no | | [instance\_type](#input\_instance\_type) | The type of the instance to launch | `string` | `""` | no | | [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | | [key\_name](#input\_key\_name) | The key name that should be used for the instance | `string` | `null` | no | @@ -134,7 +134,7 @@ module "self_managed_node_group" { | [launch\_template\_tags](#input\_launch\_template\_tags) | A map of additional tags to add to the tag\_specifications of launch template created | `map(string)` | `{}` | no | | [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | | [launch\_template\_version](#input\_launch\_template\_version) | Launch template version. Can be version number, `$Latest`, or `$Default` | `string` | `null` | no | -| [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with | `map(string)` | `null` | no | +| [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with | `map(string)` | `{}` | no | | [max\_instance\_lifetime](#input\_max\_instance\_lifetime) | The maximum amount of time, in seconds, that an instance can be in service, values must be either equal to 0 or between 604800 and 31536000 seconds | `number` | `null` | no | | [max\_size](#input\_max\_size) | The maximum size of the autoscaling group | `number` | `3` | no | | [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | @@ -144,7 +144,7 @@ module "self_managed_node_group" { | [mixed\_instances\_policy](#input\_mixed\_instances\_policy) | Configuration block containing settings to define launch targets for Auto Scaling groups | `any` | `null` | no | | [name](#input\_name) | Name of the Self managed Node Group | `string` | `""` | no | | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | -| [placement](#input\_placement) | The placement of the instance | `map(string)` | `null` | no | +| [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | | [placement\_group](#input\_placement\_group) | The name of the placement group into which you'll launch your instances, if any | `string` | `null` | no | | [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | `string` | `"linux"` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | @@ -159,10 +159,10 @@ module "self_managed_node_group" { | [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `string` | `true` | no | | [service\_linked\_role\_arn](#input\_service\_linked\_role\_arn) | The ARN of the service-linked role that the ASG will use to call other AWS services | `string` | `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones` | `list(string)` | `null` | no | -| [suspended\_processes](#input\_suspended\_processes) | A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly | `list(string)` | `null` | no | +| [suspended\_processes](#input\_suspended\_processes) | A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly | `list(string)` | `[]` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [target\_group\_arns](#input\_target\_group\_arns) | A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing | `list(string)` | `[]` | no | -| [termination\_policies](#input\_termination\_policies) | A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default` | `list(string)` | `null` | no | +| [termination\_policies](#input\_termination\_policies) | A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default` | `list(string)` | `[]` | no | | [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update Default Version each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | | [use\_default\_tags](#input\_use\_default\_tags) | Enables/disables the use of provider default tags in the tag\_specifications of the Auto Scaling group | `bool` | `false` | no | | [use\_mixed\_instances\_policy](#input\_use\_mixed\_instances\_policy) | Determines whether to use a mixed instances policy in the autoscaling group or not | `bool` | `false` | no | @@ -172,7 +172,7 @@ module "self_managed_node_group" { | [vpc\_security\_group\_ids](#input\_vpc\_security\_group\_ids) | A list of security group IDs to associate | `list(string)` | `[]` | no | | [wait\_for\_capacity\_timeout](#input\_wait\_for\_capacity\_timeout) | A maximum duration that Terraform should wait for ASG instances to be healthy before timing out. (See also Waiting for Capacity below.) Setting this to '0' causes Terraform to skip all Capacity Waiting behavior. | `string` | `null` | no | | [wait\_for\_elb\_capacity](#input\_wait\_for\_elb\_capacity) | Setting this will cause Terraform to wait for exactly this number of healthy instances in all attached load balancers on both create and update operations. Takes precedence over `min_elb_capacity` behavior. | `number` | `null` | no | -| [warm\_pool](#input\_warm\_pool) | If this block is configured, add a Warm Pool to the specified Auto Scaling group | `any` | `null` | no | +| [warm\_pool](#input\_warm\_pool) | If this block is configured, add a Warm Pool to the specified Auto Scaling group | `any` | `{}` | no | ## Outputs diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 909910ff05..f74fb8189e 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -94,7 +94,7 @@ resource "aws_launch_template" "this" { } dynamic "capacity_reservation_specification" { - for_each = var.capacity_reservation_specification != null ? [var.capacity_reservation_specification] : [] + for_each = length(var.capacity_reservation_specification) > 0 ? [var.capacity_reservation_specification] : [] content { capacity_reservation_preference = lookup(capacity_reservation_specification.value, "capacity_reservation_preference", null) @@ -108,7 +108,7 @@ resource "aws_launch_template" "this" { } dynamic "cpu_options" { - for_each = var.cpu_options != null ? [var.cpu_options] : [] + for_each = length(var.cpu_options) > 0 ? [var.cpu_options] : [] content { core_count = cpu_options.value.core_count threads_per_core = cpu_options.value.threads_per_core @@ -116,35 +116,35 @@ resource "aws_launch_template" "this" { } dynamic "credit_specification" { - for_each = var.credit_specification != null ? [var.credit_specification] : [] + for_each = length(var.credit_specification) > 0 ? [var.credit_specification] : [] content { cpu_credits = credit_specification.value.cpu_credits } } dynamic "elastic_gpu_specifications" { - for_each = var.elastic_gpu_specifications != null ? [var.elastic_gpu_specifications] : [] + for_each = length(var.elastic_gpu_specifications) > 0 ? [var.elastic_gpu_specifications] : [] content { type = elastic_gpu_specifications.value.type } } dynamic "elastic_inference_accelerator" { - for_each = var.elastic_inference_accelerator != null ? [var.elastic_inference_accelerator] : [] + for_each = length(var.elastic_inference_accelerator) > 0 ? [var.elastic_inference_accelerator] : [] content { type = elastic_inference_accelerator.value.type } } dynamic "enclave_options" { - for_each = var.enclave_options != null ? [var.enclave_options] : [] + for_each = length(var.enclave_options) > 0 ? [var.enclave_options] : [] content { enabled = enclave_options.value.enabled } } dynamic "hibernation_options" { - for_each = var.hibernation_options != null ? [var.hibernation_options] : [] + for_each = length(var.hibernation_options) > 0 ? [var.hibernation_options] : [] content { configured = hibernation_options.value.configured } @@ -155,12 +155,12 @@ resource "aws_launch_template" "this" { } dynamic "instance_market_options" { - for_each = var.instance_market_options != null ? [var.instance_market_options] : [] + for_each = length(var.instance_market_options) > 0 ? [var.instance_market_options] : [] content { market_type = instance_market_options.value.market_type dynamic "spot_options" { - for_each = lookup(instance_market_options.value, "spot_options", null) != null ? [instance_market_options.value.spot_options] : [] + for_each = length(lookup(instance_market_options.value, "spot_options", {})) > 0 ? [instance_market_options.value.spot_options] : [] content { block_duration_minutes = lookup(spot_options.value, "block_duration_minutes", null) instance_interruption_behavior = lookup(spot_options.value, "instance_interruption_behavior", null) @@ -173,14 +173,14 @@ resource "aws_launch_template" "this" { } dynamic "license_specification" { - for_each = var.license_specifications != null ? [var.license_specifications] : [] + for_each = length(var.license_specifications) > 0 ? [var.license_specifications] : [] content { license_configuration_arn = license_specifications.value.license_configuration_arn } } dynamic "metadata_options" { - for_each = var.metadata_options != null ? [var.metadata_options] : [] + for_each = length(var.metadata_options) > 0 ? [var.metadata_options] : [] content { http_endpoint = lookup(metadata_options.value, "http_endpoint", null) http_tokens = lookup(metadata_options.value, "http_tokens", null) @@ -218,7 +218,7 @@ resource "aws_launch_template" "this" { } dynamic "placement" { - for_each = var.placement != null ? [var.placement] : [] + for_each = length(var.placement) > 0 ? [var.placement] : [] content { affinity = lookup(placement.value, "affinity", null) availability_zone = lookup(placement.value, "availability_zone", null) @@ -318,13 +318,13 @@ resource "aws_autoscaling_group" "this" { } dynamic "instance_refresh" { - for_each = var.instance_refresh != null ? [var.instance_refresh] : [] + for_each = length(var.instance_refresh) > 0 ? [var.instance_refresh] : [] content { strategy = instance_refresh.value.strategy triggers = lookup(instance_refresh.value, "triggers", null) dynamic "preferences" { - for_each = lookup(instance_refresh.value, "preferences", null) != null ? [instance_refresh.value.preferences] : [] + for_each = length(lookup(instance_refresh.value, "preferences", {})) > 0 ? [instance_refresh.value.preferences] : [] content { instance_warmup = lookup(preferences.value, "instance_warmup", null) min_healthy_percentage = lookup(preferences.value, "min_healthy_percentage", null) @@ -363,7 +363,7 @@ resource "aws_autoscaling_group" "this" { weighted_capacity = lookup(override.value, "weighted_capacity", null) dynamic "launch_template_specification" { - for_each = lookup(override.value, "launch_template_specification", null) != null ? override.value.launch_template_specification : [] + for_each = length(lookup(override.value, "launch_template_specification", {})) > 0 ? override.value.launch_template_specification : [] content { launch_template_id = lookup(launch_template_specification.value, "launch_template_id", null) } @@ -375,7 +375,7 @@ resource "aws_autoscaling_group" "this" { } dynamic "warm_pool" { - for_each = var.warm_pool != null ? [var.warm_pool] : [] + for_each = length(var.warm_pool) > 0 ? [var.warm_pool] : [] content { pool_state = lookup(warm_pool.value, "pool_state", null) min_size = lookup(warm_pool.value, "min_size", null) diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index d3fcd726c2..5aa20066c5 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -135,55 +135,55 @@ variable "block_device_mappings" { variable "capacity_reservation_specification" { description = "Targeting for EC2 capacity reservations" type = any - default = null + default = {} } variable "cpu_options" { description = "The CPU options for the instance" type = map(string) - default = null + default = {} } variable "credit_specification" { description = "Customize the credit specification of the instance" type = map(string) - default = null + default = {} } variable "elastic_gpu_specifications" { description = "The elastic GPU to attach to the instance" type = map(string) - default = null + default = {} } variable "elastic_inference_accelerator" { description = "Configuration block containing an Elastic Inference Accelerator to attach to the instance" type = map(string) - default = null + default = {} } variable "enclave_options" { description = "Enable Nitro Enclaves on launched instances" type = map(string) - default = null + default = {} } variable "hibernation_options" { description = "The hibernation options for the instance" type = map(string) - default = null + default = {} } variable "instance_market_options" { description = "The market (purchasing) option for the instance" type = any - default = null + default = {} } variable "license_specifications" { description = "A list of license specifications to associate with" type = map(string) - default = null + default = {} } variable "network_interfaces" { @@ -195,7 +195,7 @@ variable "network_interfaces" { variable "placement" { description = "The placement of the instance" type = map(string) - default = null + default = {} } variable "ebs_optimized" { @@ -389,13 +389,13 @@ variable "force_delete" { variable "termination_policies" { description = "A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default`" type = list(string) - default = null + default = [] } variable "suspended_processes" { description = "A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly" type = list(string) - default = null + default = [] } variable "max_instance_lifetime" { @@ -407,7 +407,7 @@ variable "max_instance_lifetime" { variable "enabled_metrics" { description = "A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances`" type = list(string) - default = null + default = [] } variable "metrics_granularity" { @@ -431,7 +431,7 @@ variable "initial_lifecycle_hooks" { variable "instance_refresh" { description = "If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated" type = any - default = null + default = {} } variable "use_mixed_instances_policy" { @@ -449,7 +449,7 @@ variable "mixed_instances_policy" { variable "warm_pool" { description = "If this block is configured, add a Warm Pool to the specified Auto Scaling group" type = any - default = null + default = {} } variable "delete_timeout" { diff --git a/node_groups.tf b/node_groups.tf index 8c0916ade3..76d2880323 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -218,7 +218,7 @@ module "fargate_profile" { iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.fargate_profile_defaults.iam_role_permissions_boundary, null) iam_role_tags = try(each.value.iam_role_tags, var.fargate_profile_defaults.iam_role_tags, {}) iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.fargate_profile_defaults.iam_role_attach_cni_policy, true) - iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.fargate_profile_defaults.iam_role_additional_policies, []) + iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.fargate_profile_defaults.iam_role_additional_policies, {}) tags = merge(var.tags, try(each.value.tags, var.fargate_profile_defaults.tags, {})) } @@ -292,18 +292,18 @@ module "eks_managed_node_group" { ram_disk_id = try(each.value.ram_disk_id, var.eks_managed_node_group_defaults.ram_disk_id, null) block_device_mappings = try(each.value.block_device_mappings, var.eks_managed_node_group_defaults.block_device_mappings, {}) - capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.eks_managed_node_group_defaults.capacity_reservation_specification, null) - cpu_options = try(each.value.cpu_options, var.eks_managed_node_group_defaults.cpu_options, null) - credit_specification = try(each.value.credit_specification, var.eks_managed_node_group_defaults.credit_specification, null) - elastic_gpu_specifications = try(each.value.elastic_gpu_specifications, var.eks_managed_node_group_defaults.elastic_gpu_specifications, null) - elastic_inference_accelerator = try(each.value.elastic_inference_accelerator, var.eks_managed_node_group_defaults.elastic_inference_accelerator, null) - enclave_options = try(each.value.enclave_options, var.eks_managed_node_group_defaults.enclave_options, null) - instance_market_options = try(each.value.instance_market_options, var.eks_managed_node_group_defaults.instance_market_options, null) - license_specifications = try(each.value.license_specifications, var.eks_managed_node_group_defaults.license_specifications, null) + capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.eks_managed_node_group_defaults.capacity_reservation_specification, {}) + cpu_options = try(each.value.cpu_options, var.eks_managed_node_group_defaults.cpu_options, {}) + credit_specification = try(each.value.credit_specification, var.eks_managed_node_group_defaults.credit_specification, {}) + elastic_gpu_specifications = try(each.value.elastic_gpu_specifications, var.eks_managed_node_group_defaults.elastic_gpu_specifications, {}) + elastic_inference_accelerator = try(each.value.elastic_inference_accelerator, var.eks_managed_node_group_defaults.elastic_inference_accelerator, {}) + enclave_options = try(each.value.enclave_options, var.eks_managed_node_group_defaults.enclave_options, {}) + instance_market_options = try(each.value.instance_market_options, var.eks_managed_node_group_defaults.instance_market_options, {}) + license_specifications = try(each.value.license_specifications, var.eks_managed_node_group_defaults.license_specifications, {}) metadata_options = try(each.value.metadata_options, var.eks_managed_node_group_defaults.metadata_options, local.metadata_options) enable_monitoring = try(each.value.enable_monitoring, var.eks_managed_node_group_defaults.enable_monitoring, true) network_interfaces = try(each.value.network_interfaces, var.eks_managed_node_group_defaults.network_interfaces, []) - placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, null) + placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, {}) # IAM role create_iam_role = try(each.value.create_iam_role, var.eks_managed_node_group_defaults.create_iam_role, true) @@ -364,28 +364,28 @@ module "self_managed_node_group" { default_cooldown = try(each.value.default_cooldown, var.self_managed_node_group_defaults.default_cooldown, null) protect_from_scale_in = try(each.value.protect_from_scale_in, var.self_managed_node_group_defaults.protect_from_scale_in, null) - target_group_arns = try(each.value.target_group_arns, var.self_managed_node_group_defaults.target_group_arns, null) + target_group_arns = try(each.value.target_group_arns, var.self_managed_node_group_defaults.target_group_arns, []) placement_group = try(each.value.placement_group, var.self_managed_node_group_defaults.placement_group, null) health_check_type = try(each.value.health_check_type, var.self_managed_node_group_defaults.health_check_type, null) health_check_grace_period = try(each.value.health_check_grace_period, var.self_managed_node_group_defaults.health_check_grace_period, null) force_delete = try(each.value.force_delete, var.self_managed_node_group_defaults.force_delete, null) - termination_policies = try(each.value.termination_policies, var.self_managed_node_group_defaults.termination_policies, null) - suspended_processes = try(each.value.suspended_processes, var.self_managed_node_group_defaults.suspended_processes, null) + termination_policies = try(each.value.termination_policies, var.self_managed_node_group_defaults.termination_policies, []) + suspended_processes = try(each.value.suspended_processes, var.self_managed_node_group_defaults.suspended_processes, []) max_instance_lifetime = try(each.value.max_instance_lifetime, var.self_managed_node_group_defaults.max_instance_lifetime, null) - enabled_metrics = try(each.value.enabled_metrics, var.self_managed_node_group_defaults.enabled_metrics, null) + enabled_metrics = try(each.value.enabled_metrics, var.self_managed_node_group_defaults.enabled_metrics, []) metrics_granularity = try(each.value.metrics_granularity, var.self_managed_node_group_defaults.metrics_granularity, null) service_linked_role_arn = try(each.value.service_linked_role_arn, var.self_managed_node_group_defaults.service_linked_role_arn, null) initial_lifecycle_hooks = try(each.value.initial_lifecycle_hooks, var.self_managed_node_group_defaults.initial_lifecycle_hooks, []) - instance_refresh = try(each.value.instance_refresh, var.self_managed_node_group_defaults.instance_refresh, null) + instance_refresh = try(each.value.instance_refresh, var.self_managed_node_group_defaults.instance_refresh, {}) use_mixed_instances_policy = try(each.value.use_mixed_instances_policy, var.self_managed_node_group_defaults.use_mixed_instances_policy, false) mixed_instances_policy = try(each.value.mixed_instances_policy, var.self_managed_node_group_defaults.mixed_instances_policy, null) - warm_pool = try(each.value.warm_pool, var.self_managed_node_group_defaults.warm_pool, null) + warm_pool = try(each.value.warm_pool, var.self_managed_node_group_defaults.warm_pool, {}) create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, false) - schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, null) + schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, {}) delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) use_default_tags = try(each.value.use_default_tags, var.self_managed_node_group_defaults.use_default_tags, false) @@ -421,20 +421,20 @@ module "self_managed_node_group" { kernel_id = try(each.value.kernel_id, var.self_managed_node_group_defaults.kernel_id, null) ram_disk_id = try(each.value.ram_disk_id, var.self_managed_node_group_defaults.ram_disk_id, null) - block_device_mappings = try(each.value.block_device_mappings, var.self_managed_node_group_defaults.block_device_mappings, []) - capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.self_managed_node_group_defaults.capacity_reservation_specification, null) - cpu_options = try(each.value.cpu_options, var.self_managed_node_group_defaults.cpu_options, null) - credit_specification = try(each.value.credit_specification, var.self_managed_node_group_defaults.credit_specification, null) - elastic_gpu_specifications = try(each.value.elastic_gpu_specifications, var.self_managed_node_group_defaults.elastic_gpu_specifications, null) - elastic_inference_accelerator = try(each.value.elastic_inference_accelerator, var.self_managed_node_group_defaults.elastic_inference_accelerator, null) - enclave_options = try(each.value.enclave_options, var.self_managed_node_group_defaults.enclave_options, null) - hibernation_options = try(each.value.hibernation_options, var.self_managed_node_group_defaults.hibernation_options, null) - instance_market_options = try(each.value.instance_market_options, var.self_managed_node_group_defaults.instance_market_options, null) - license_specifications = try(each.value.license_specifications, var.self_managed_node_group_defaults.license_specifications, null) + block_device_mappings = try(each.value.block_device_mappings, var.self_managed_node_group_defaults.block_device_mappings, {}) + capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.self_managed_node_group_defaults.capacity_reservation_specification, {}) + cpu_options = try(each.value.cpu_options, var.self_managed_node_group_defaults.cpu_options, {}) + credit_specification = try(each.value.credit_specification, var.self_managed_node_group_defaults.credit_specification, {}) + elastic_gpu_specifications = try(each.value.elastic_gpu_specifications, var.self_managed_node_group_defaults.elastic_gpu_specifications, {}) + elastic_inference_accelerator = try(each.value.elastic_inference_accelerator, var.self_managed_node_group_defaults.elastic_inference_accelerator, {}) + enclave_options = try(each.value.enclave_options, var.self_managed_node_group_defaults.enclave_options, {}) + hibernation_options = try(each.value.hibernation_options, var.self_managed_node_group_defaults.hibernation_options, {}) + instance_market_options = try(each.value.instance_market_options, var.self_managed_node_group_defaults.instance_market_options, {}) + license_specifications = try(each.value.license_specifications, var.self_managed_node_group_defaults.license_specifications, {}) metadata_options = try(each.value.metadata_options, var.self_managed_node_group_defaults.metadata_options, local.metadata_options) enable_monitoring = try(each.value.enable_monitoring, var.self_managed_node_group_defaults.enable_monitoring, true) network_interfaces = try(each.value.network_interfaces, var.self_managed_node_group_defaults.network_interfaces, []) - placement = try(each.value.placement, var.self_managed_node_group_defaults.placement, null) + placement = try(each.value.placement, var.self_managed_node_group_defaults.placement, {}) # IAM role create_iam_instance_profile = try(each.value.create_iam_instance_profile, var.self_managed_node_group_defaults.create_iam_instance_profile, true) From 661db03fb5aab04bc72916e60da1ac63bc6bf91a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 1 Jul 2022 11:54:25 +0000 Subject: [PATCH 567/962] chore(release): version 18.26.2 [skip ci] ### [18.26.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.1...v18.26.2) (2022-07-01) ### Bug Fixes * Correct variable types to improve dynamic check correctness ([#2133](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2133)) ([2d7701c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/2d7701c3b0f2c6dcc10f31fc1f703bfde31b2c5b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a38ae86a05..fa999ee1af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.26.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.1...v18.26.2) (2022-07-01) + + +### Bug Fixes + +* Correct variable types to improve dynamic check correctness ([#2133](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2133)) ([2d7701c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/2d7701c3b0f2c6dcc10f31fc1f703bfde31b2c5b)) + ### [18.26.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.0...v18.26.1) (2022-06-29) From 76669d3aac7d43008e8a8f35c4df394db18a4d4e Mon Sep 17 00:00:00 2001 From: Ilya Kisleyko Date: Tue, 5 Jul 2022 22:02:45 +0400 Subject: [PATCH 568/962] docs: Fix links in `docs/compute_resources.md` table of contents (#2144) --- docs/compute_resources.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 2fefd17e9b..51d41025bb 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -2,10 +2,10 @@ ## Table of Contents -- [EKS Managed Node Groups](https://github.com/terraform-aws-module/terraform-aws-eks/blob/master/docs/node_groups.md#eks-managed-node-groups) -- [Self Managed Node Groups](https://github.com/terraform-aws-module/terraform-aws-eks/blob/master/docs/node_groups.md#self-managed-node-groups) -- [Fargate Profiles](https://github.com/terraform-aws-module/terraform-aws-eks/blob/master/docs/node_groups.md#fargate-profiles) -- [Default Configurations](https://github.com/terraform-aws-module/terraform-aws-eks/blob/master/docs/node_groups.md#default-configurations) +- [EKS Managed Node Groups](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/compute_resources.md#eks-managed-node-groups) +- [Self Managed Node Groups](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/compute_resources.md#self-managed-node-groups) +- [Fargate Profiles](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/compute_resources.md#fargate-profiles) +- [Default Configurations](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/compute_resources.md#default-configurations) ℹ️ Only the pertinent attributes are shown below for brevity From c4e6d28fc064435f6f05c6c57d7fff8576d9fbba Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 5 Jul 2022 14:06:25 -0400 Subject: [PATCH 569/962] fix: Correct Fargate profiles additional IAM role policies default type to match variable (#2143) --- node_groups.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_groups.tf b/node_groups.tf index 76d2880323..ca51f32c8c 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -218,7 +218,7 @@ module "fargate_profile" { iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.fargate_profile_defaults.iam_role_permissions_boundary, null) iam_role_tags = try(each.value.iam_role_tags, var.fargate_profile_defaults.iam_role_tags, {}) iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.fargate_profile_defaults.iam_role_attach_cni_policy, true) - iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.fargate_profile_defaults.iam_role_additional_policies, {}) + iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.fargate_profile_defaults.iam_role_additional_policies, []) tags = merge(var.tags, try(each.value.tags, var.fargate_profile_defaults.tags, {})) } From 5955d26589979abdad832e4e9a2dec9753ebc738 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 5 Jul 2022 18:06:59 +0000 Subject: [PATCH 570/962] chore(release): version 18.26.3 [skip ci] ### [18.26.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.2...v18.26.3) (2022-07-05) ### Bug Fixes * Correct Fargate profiles additional IAM role policies default type to match variable ([#2143](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2143)) ([c4e6d28](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c4e6d28fc064435f6f05c6c57d7fff8576d9fbba)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa999ee1af..a493f03b31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.26.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.2...v18.26.3) (2022-07-05) + + +### Bug Fixes + +* Correct Fargate profiles additional IAM role policies default type to match variable ([#2143](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2143)) ([c4e6d28](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c4e6d28fc064435f6f05c6c57d7fff8576d9fbba)) + ### [18.26.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.1...v18.26.2) (2022-07-01) From 62dddaedb997758348f9c0324a170c0e11e1d692 Mon Sep 17 00:00:00 2001 From: KevinSnyderCodes Date: Wed, 6 Jul 2022 13:06:08 -0700 Subject: [PATCH 571/962] docs: Add fix for `KUBELET_EXTRA_ARGS` overwrite to EKS managed node group example (#2150) --- examples/eks_managed_node_group/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 71eb4fa26d..96e79483e3 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -34,6 +34,7 @@ When using the default AMI provided by the EKS Managed Node Group service (i.e. EOF # Source extra environment variables in bootstrap script sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + sed -i 's/KUBELET_EXTRA_ARGS=$2/KUBELET_EXTRA_ARGS="$2 $KUBELET_EXTRA_ARGS"/' /etc/eks/bootstrap.sh EOT } ... From 312e4a4d59cb10a762a4045e9944f3f837126933 Mon Sep 17 00:00:00 2001 From: Eduardo Garibay-Frausto Date: Thu, 7 Jul 2022 11:44:36 -0400 Subject: [PATCH 572/962] docs: Correct example typo for NTP address (#2154) --- examples/eks_managed_node_group/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index b8f7fd60ab..ef702dff87 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -92,7 +92,7 @@ module "eks" { } # Extend node-to-node security group rules - node_security_group_ntp_ipv4_cidr_block = ["fd00:ec2::123/128"] + node_security_group_ntp_ipv6_cidr_block = ["fd00:ec2::123/128"] node_security_group_additional_rules = { ingress_self_all = { description = "Node to node all ports/protocols" From 67e0f752b9910c0124afb5409cdf4b32fd0087a0 Mon Sep 17 00:00:00 2001 From: Gaston Festari Date: Thu, 7 Jul 2022 20:09:02 -0300 Subject: [PATCH 573/962] docs: EKS Managed Node Groups example typo (#2152) --- examples/eks_managed_node_group/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 96e79483e3..33016c0e85 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -14,7 +14,7 @@ See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/man ## Container Runtime & User Data -When using the default AMI provided by the EKS Managed Node Group service (i.e. - not specifying a value for `ami_id`), users should be aware of the limitations of configuring the node bootstrap process via user data. Due to not having direct access to the bootrap.sh script invocation and therefore its configuration flags (this is provide by the EKS Managed Node Group service in the node user data), a work around for ensuring the appropriate configuration settings is shown below. The following example shows how to inject configuration variables ahead of the merged user data provided by the EKS Managed Node Group service as well as how to enable the containerd runtime using this approach. More details can be found [here](https://github.com/awslabs/amazon-eks-ami/issues/844). +When using the default AMI provided by the EKS Managed Node Group service (i.e. - not specifying a value for `ami_id`), users should be aware of the limitations of configuring the node bootstrap process via user data. Due to not having direct access to the bootrap.sh script invocation and therefore its configuration flags (this is provided by the EKS Managed Node Group service in the node user data), a workaround for ensuring the appropriate configuration settings is shown below. The following example shows how to inject configuration variables ahead of the merged user data provided by the EKS Managed Node Group service as well as how to enable the containerd runtime using this approach. More details can be found [here](https://github.com/awslabs/amazon-eks-ami/issues/844). ```hcl ... From f2d67ffa97cc0f9827f75673b1cd263e3a5062b6 Mon Sep 17 00:00:00 2001 From: Ian Lunam Date: Wed, 20 Jul 2022 23:01:07 +1200 Subject: [PATCH 574/962] fix: Use partition data source on VPC CNI IPv6 policy (#2161) --- node_groups.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_groups.tf b/node_groups.tf index ca51f32c8c..402191f162 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -30,7 +30,7 @@ data "aws_iam_policy_document" "cni_ipv6_policy" { statement { sid = "CreateTags" actions = ["ec2:CreateTags"] - resources = ["arn:aws:ec2:*:*:network-interface/*"] + resources = ["arn:${data.aws_partition.current.partition}:ec2:*:*:network-interface/*"] } } From f8d4f3bdbc832b189ef037fc21e9830787e15c65 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 20 Jul 2022 11:02:06 +0000 Subject: [PATCH 575/962] chore(release): version 18.26.4 [skip ci] ### [18.26.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.3...v18.26.4) (2022-07-20) ### Bug Fixes * Use partition data source on VPC CNI IPv6 policy ([#2161](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2161)) ([f2d67ff](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f2d67ffa97cc0f9827f75673b1cd263e3a5062b6)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a493f03b31..2caea1c833 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.26.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.3...v18.26.4) (2022-07-20) + + +### Bug Fixes + +* Use partition data source on VPC CNI IPv6 policy ([#2161](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2161)) ([f2d67ff](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f2d67ffa97cc0f9827f75673b1cd263e3a5062b6)) + ### [18.26.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.2...v18.26.3) (2022-07-05) From 0fd1ab1db9b752e58211428e3c19f62655e5f97d Mon Sep 17 00:00:00 2001 From: Pedro Fragoso Date: Wed, 20 Jul 2022 19:05:47 +0100 Subject: [PATCH 576/962] fix: Bump kms module to 1.0.2 to fix malformed policy document when not specifying key_owners (#2163) --- README.md | 2 +- main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4c90720368..50eb79eb08 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|--------|---------| | [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ./modules/eks-managed-node-group | n/a | | [fargate\_profile](#module\_fargate\_profile) | ./modules/fargate-profile | n/a | -| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 1.0.1 | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 1.0.2 | | [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ./modules/self-managed-node-group | n/a | ## Resources diff --git a/main.tf b/main.tf index 0924a6e8f3..82f3f7a234 100644 --- a/main.tf +++ b/main.tf @@ -89,7 +89,7 @@ resource "aws_cloudwatch_log_group" "this" { module "kms" { source = "terraform-aws-modules/kms/aws" - version = "1.0.1" # Note - be mindful of Terraform/provider version compatibility between modules + version = "1.0.2" # Note - be mindful of Terraform/provider version compatibility between modules create = var.create_kms_key From 640a99485eff31dfb5c671177c04d42023772260 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 20 Jul 2022 18:06:27 +0000 Subject: [PATCH 577/962] chore(release): version 18.26.5 [skip ci] ### [18.26.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.4...v18.26.5) (2022-07-20) ### Bug Fixes * Bump kms module to 1.0.2 to fix malformed policy document when not specifying key_owners ([#2163](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2163)) ([0fd1ab1](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0fd1ab1db9b752e58211428e3c19f62655e5f97d)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2caea1c833..ca907035a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.26.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.4...v18.26.5) (2022-07-20) + + +### Bug Fixes + +* Bump kms module to 1.0.2 to fix malformed policy document when not specifying key_owners ([#2163](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2163)) ([0fd1ab1](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0fd1ab1db9b752e58211428e3c19f62655e5f97d)) + ### [18.26.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.3...v18.26.4) (2022-07-20) From d990ea8aff682315828d7c177a309c71541e023c Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 22 Jul 2022 08:29:00 -0400 Subject: [PATCH 578/962] fix: Pin TLS provider version to 3.x versions only (#2174) --- .pre-commit-config.yaml | 2 +- README.md | 4 ++-- examples/eks_managed_node_group/README.md | 4 ++-- examples/eks_managed_node_group/versions.tf | 2 +- examples/self_managed_node_group/README.md | 4 ++-- examples/self_managed_node_group/versions.tf | 2 +- main.tf | 2 +- versions.tf | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9eaabc7a07..727e21c06e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.73.0 + rev: v1.74.1 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/README.md b/README.md index 50eb79eb08..9a80aa5df5 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | -| [tls](#requirement\_tls) | >= 3.0 | +| [tls](#requirement\_tls) | ~> 3.0 | ## Providers @@ -218,7 +218,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|---------| | [aws](#provider\_aws) | >= 3.72 | | [kubernetes](#provider\_kubernetes) | >= 2.10 | -| [tls](#provider\_tls) | >= 3.0 | +| [tls](#provider\_tls) | ~> 3.0 | ## Modules diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 33016c0e85..ea51a159cb 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -60,14 +60,14 @@ Note that this example may create resources which cost money. Run `terraform des | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | -| [tls](#requirement\_tls) | >= 2.2 | +| [tls](#requirement\_tls) | ~> 3.0 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | -| [tls](#provider\_tls) | >= 2.2 | +| [tls](#provider\_tls) | ~> 3.0 | ## Modules diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index 40b7e32ab9..7fa16dcd48 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -8,7 +8,7 @@ terraform { } tls = { source = "hashicorp/tls" - version = ">= 2.2" + version = "~> 3.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index a0caf4a2e2..cfdff57cb7 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -28,14 +28,14 @@ Note that this example may create resources which cost money. Run `terraform des | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | -| [tls](#requirement\_tls) | >= 2.2 | +| [tls](#requirement\_tls) | ~> 3.0 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | -| [tls](#provider\_tls) | >= 2.2 | +| [tls](#provider\_tls) | ~> 3.0 | ## Modules diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index 40b7e32ab9..7fa16dcd48 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -8,7 +8,7 @@ terraform { } tls = { source = "hashicorp/tls" - version = ">= 2.2" + version = "~> 3.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/main.tf b/main.tf index 82f3f7a234..593ab775a9 100644 --- a/main.tf +++ b/main.tf @@ -91,7 +91,7 @@ module "kms" { source = "terraform-aws-modules/kms/aws" version = "1.0.2" # Note - be mindful of Terraform/provider version compatibility between modules - create = var.create_kms_key + create = local.create && var.create_kms_key description = coalesce(var.kms_key_description, "${var.cluster_name} cluster encryption key") key_usage = "ENCRYPT_DECRYPT" diff --git a/versions.tf b/versions.tf index fde7af0f23..7fa16dcd48 100644 --- a/versions.tf +++ b/versions.tf @@ -8,7 +8,7 @@ terraform { } tls = { source = "hashicorp/tls" - version = ">= 3.0" + version = "~> 3.0" } kubernetes = { source = "hashicorp/kubernetes" From 58285f37d1dfaacbf99055ed87f1be19cc39f5ad Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 22 Jul 2022 12:29:39 +0000 Subject: [PATCH 579/962] chore(release): version 18.26.6 [skip ci] ### [18.26.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.5...v18.26.6) (2022-07-22) ### Bug Fixes * Pin TLS provider version to 3.x versions only ([#2174](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2174)) ([d990ea8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d990ea8aff682315828d7c177a309c71541e023c)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca907035a0..976d483347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.26.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.5...v18.26.6) (2022-07-22) + + +### Bug Fixes + +* Pin TLS provider version to 3.x versions only ([#2174](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2174)) ([d990ea8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d990ea8aff682315828d7c177a309c71541e023c)) + ### [18.26.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.4...v18.26.5) (2022-07-20) From 6d3fc1b93a68be590d37e682ab4e70ad0e2a6f8f Mon Sep 17 00:00:00 2001 From: pen-pal <61139563+pen-pal@users.noreply.github.com> Date: Tue, 9 Aug 2022 05:46:07 +0545 Subject: [PATCH 580/962] docs: Update KMS Key Default Policy Description (#2176) --- README.md | 2 +- variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9a80aa5df5..3efc1e98fd 100644 --- a/README.md +++ b/README.md @@ -324,7 +324,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [kms\_key\_aliases](#input\_kms\_key\_aliases) | A list of aliases to create. Note - due to the use of `toset()`, values must be static strings and not computed values | `list(string)` | `[]` | no | | [kms\_key\_deletion\_window\_in\_days](#input\_kms\_key\_deletion\_window\_in\_days) | The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key. If you specify a value, it must be between `7` and `30`, inclusive. If you do not specify a value, it defaults to `30` | `number` | `null` | no | | [kms\_key\_description](#input\_kms\_key\_description) | The description of the key as viewed in AWS console | `string` | `null` | no | -| [kms\_key\_enable\_default\_policy](#input\_kms\_key\_enable\_default\_policy) | Specifies whether to enable the default key policy. Defaults to `true` | `bool` | `false` | no | +| [kms\_key\_enable\_default\_policy](#input\_kms\_key\_enable\_default\_policy) | Specifies whether to enable the default key policy. Defaults to `false` | `bool` | `false` | no | | [kms\_key\_override\_policy\_documents](#input\_kms\_key\_override\_policy\_documents) | List of IAM policy documents that are merged together into the exported document. In merging, statements with non-blank `sid`s will override statements with the same `sid` | `list(string)` | `[]` | no | | [kms\_key\_owners](#input\_kms\_key\_owners) | A list of IAM ARNs for those who will have full key permissions (`kms:*`) | `list(string)` | `[]` | no | | [kms\_key\_service\_users](#input\_kms\_key\_service\_users) | A list of IAM ARNs for [key service users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-service-integration) | `list(string)` | `[]` | no | diff --git a/variables.tf b/variables.tf index ad7777063f..c5bde1abb5 100644 --- a/variables.tf +++ b/variables.tf @@ -145,7 +145,7 @@ variable "enable_kms_key_rotation" { } variable "kms_key_enable_default_policy" { - description = "Specifies whether to enable the default key policy. Defaults to `true`" + description = "Specifies whether to enable the default key policy. Defaults to `false`" type = bool default = false } From 93065fabdf508267b399f677d561f18fd6d7b7f0 Mon Sep 17 00:00:00 2001 From: Igor Brites Date: Tue, 9 Aug 2022 15:06:39 -0400 Subject: [PATCH 581/962] feat: Default to clusters OIDC issuer URL for `aws_eks_identity_provider_config` (#2190) feat: setting oidc issuer as default option for aws_eks_identity_provider_config Co-authored-by: Bryant Biggs --- examples/complete/main.tf | 7 +++++++ main.tf | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 55fc8bf38c..112e3b531d 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -196,6 +196,13 @@ module "eks" { } } + # OIDC Identity provider + cluster_identity_providers = { + sts = { + client_id = "sts.amazonaws.com" + } + } + # aws-auth configmap manage_aws_auth_configmap = true diff --git a/main.tf b/main.tf index 593ab775a9..6b0dc66ecc 100644 --- a/main.tf +++ b/main.tf @@ -369,7 +369,7 @@ resource "aws_eks_identity_provider_config" "this" { groups_claim = lookup(each.value, "groups_claim", null) groups_prefix = lookup(each.value, "groups_prefix", null) identity_provider_config_name = try(each.value.identity_provider_config_name, each.key) - issuer_url = each.value.issuer_url + issuer_url = try(each.value.issuer_url, aws_eks_cluster.this[0].identity[0].oidc[0].issuer) required_claims = lookup(each.value, "required_claims", null) username_claim = lookup(each.value, "username_claim", null) username_prefix = lookup(each.value, "username_prefix", null) From 34dc4bd37d2766eb35e31081fefc537ca5db72ac Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 9 Aug 2022 19:07:15 +0000 Subject: [PATCH 582/962] chore(release): version 18.27.0 [skip ci] ## [18.27.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.6...v18.27.0) (2022-08-09) ### Features * Default to clusters OIDC issuer URL for `aws_eks_identity_provider_config` ([#2190](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2190)) ([93065fa](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/93065fabdf508267b399f677d561f18fd6d7b7f0)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 976d483347..cf50d984be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.27.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.6...v18.27.0) (2022-08-09) + + +### Features + +* Default to clusters OIDC issuer URL for `aws_eks_identity_provider_config` ([#2190](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2190)) ([93065fa](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/93065fabdf508267b399f677d561f18fd6d7b7f0)) + ### [18.26.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.5...v18.26.6) (2022-07-22) From d2f162b190596756f1bc9d8f8061e68329c3e5c4 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 9 Aug 2022 15:25:11 -0400 Subject: [PATCH 583/962] fix: Remove empty `""` from node group names output when node group creation is disabled (#2197) --- outputs.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/outputs.tf b/outputs.tf index 8ff30fc623..25b8a0b24a 100644 --- a/outputs.tf +++ b/outputs.tf @@ -179,7 +179,7 @@ output "eks_managed_node_groups" { output "eks_managed_node_groups_autoscaling_group_names" { description = "List of the autoscaling group names created by EKS managed node groups" - value = flatten([for group in module.eks_managed_node_group : group.node_group_autoscaling_group_names]) + value = compact(flatten([for group in module.eks_managed_node_group : group.node_group_autoscaling_group_names])) } ################################################################################ @@ -193,7 +193,7 @@ output "self_managed_node_groups" { output "self_managed_node_groups_autoscaling_group_names" { description = "List of the autoscaling group names created by self-managed node groups" - value = [for group in module.self_managed_node_group : group.autoscaling_group_name] + value = compact([for group in module.self_managed_node_group : group.autoscaling_group_name]) } ################################################################################ From c0f14fbdeffaee36e865e90328e633b73c4e2e10 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 9 Aug 2022 19:25:53 +0000 Subject: [PATCH 584/962] chore(release): version 18.27.1 [skip ci] ### [18.27.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.27.0...v18.27.1) (2022-08-09) ### Bug Fixes * Remove empty `""` from node group names output when node group creation is disabled ([#2197](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2197)) ([d2f162b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d2f162b190596756f1bc9d8f8061e68329c3e5c4)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf50d984be..2ff6863cd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.27.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.27.0...v18.27.1) (2022-08-09) + + +### Bug Fixes + +* Remove empty `""` from node group names output when node group creation is disabled ([#2197](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2197)) ([d2f162b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d2f162b190596756f1bc9d8f8061e68329c3e5c4)) + ## [18.27.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.26.6...v18.27.0) (2022-08-09) From 0a52d690d54a7c39fd4e0d46db36d200f7ef679e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 17 Aug 2022 06:24:22 -0400 Subject: [PATCH 585/962] feat: Add output for launch template name, and correct variable type value (#2205) --- README.md | 6 +++--- modules/eks-managed-node-group/README.md | 3 ++- modules/eks-managed-node-group/outputs.tf | 5 +++++ modules/eks-managed-node-group/variables.tf | 2 +- modules/self-managed-node-group/README.md | 3 ++- modules/self-managed-node-group/outputs.tf | 5 +++++ modules/self-managed-node-group/variables.tf | 2 +- variables.tf | 6 +++--- 8 files changed, 22 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3efc1e98fd..4ee1015837 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_encryption\_policy\_name](#input\_cluster\_encryption\_policy\_name) | Name to use on cluster encryption policy created | `string` | `null` | no | | [cluster\_encryption\_policy\_path](#input\_cluster\_encryption\_policy\_path) | Cluster encryption policy path | `string` | `null` | no | | [cluster\_encryption\_policy\_tags](#input\_cluster\_encryption\_policy\_tags) | A map of additional tags to add to the cluster encryption policy created | `map(string)` | `{}` | no | -| [cluster\_encryption\_policy\_use\_name\_prefix](#input\_cluster\_encryption\_policy\_use\_name\_prefix) | Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix | `string` | `true` | no | +| [cluster\_encryption\_policy\_use\_name\_prefix](#input\_cluster\_encryption\_policy\_use\_name\_prefix) | Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix | `bool` | `true` | no | | [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `false` | no | | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `true` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | @@ -290,7 +290,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Existing security group ID to be attached to the cluster. Required if `create_cluster_security_group` = `false` | `string` | `""` | no | | [cluster\_security\_group\_name](#input\_cluster\_security\_group\_name) | Name to use on cluster security group created | `string` | `null` | no | | [cluster\_security\_group\_tags](#input\_cluster\_security\_group\_tags) | A map of additional tags to add to the cluster security group created | `map(string)` | `{}` | no | -| [cluster\_security\_group\_use\_name\_prefix](#input\_cluster\_security\_group\_use\_name\_prefix) | Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix | `string` | `true` | no | +| [cluster\_security\_group\_use\_name\_prefix](#input\_cluster\_security\_group\_use\_name\_prefix) | Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix | `bool` | `true` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | @@ -338,7 +338,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [node\_security\_group\_ntp\_ipv4\_cidr\_block](#input\_node\_security\_group\_ntp\_ipv4\_cidr\_block) | IPv4 CIDR block to allow NTP egress. Default is public IP space, but [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) can be used as well with `["169.254.169.123/32"]` | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [node\_security\_group\_ntp\_ipv6\_cidr\_block](#input\_node\_security\_group\_ntp\_ipv6\_cidr\_block) | IPv4 CIDR block to allow NTP egress. Default is public IP space, but [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) can be used as well with `["fd00:ec2::123/128"]` | `list(string)` |
[
"::/0"
]
| no | | [node\_security\_group\_tags](#input\_node\_security\_group\_tags) | A map of additional tags to add to the node security group created | `map(string)` | `{}` | no | -| [node\_security\_group\_use\_name\_prefix](#input\_node\_security\_group\_use\_name\_prefix) | Determines whether node security group name (`node_security_group_name`) is used as a prefix | `string` | `true` | no | +| [node\_security\_group\_use\_name\_prefix](#input\_node\_security\_group\_use\_name\_prefix) | Determines whether node security group name (`node_security_group_name`) is used as a prefix | `bool` | `true` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider | `list(string)` | `[]` | no | | [prefix\_separator](#input\_prefix\_separator) | The separator to use between the prefix and the generated timestamp for resource names | `string` | `"-"` | no | | [putin\_khuylo](#input\_putin\_khuylo) | Do you agree that Putin doesn't respect Ukrainian sovereignty and territorial integrity? More info: https://en.wikipedia.org/wiki/Putin_khuylo! | `bool` | `true` | no | diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 4ec3c782c9..44aa062d4c 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -153,7 +153,7 @@ module "eks_managed_node_group" { | [security\_group\_name](#input\_security\_group\_name) | Name to use on security group created | `string` | `null` | no | | [security\_group\_rules](#input\_security\_group\_rules) | List of security group rules to add to the security group created | `any` | `{}` | no | | [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the security group created | `map(string)` | `{}` | no | -| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `string` | `true` | no | +| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `bool` | `true` | no | | [subnet\_ids](#input\_subnet\_ids) | Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME` | `list(string)` | `null` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [taints](#input\_taints) | The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group | `any` | `{}` | no | @@ -175,6 +175,7 @@ module "eks_managed_node_group" { | [launch\_template\_arn](#output\_launch\_template\_arn) | The ARN of the launch template | | [launch\_template\_id](#output\_launch\_template\_id) | The ID of the launch template | | [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | +| [launch\_template\_name](#output\_launch\_template\_name) | The name of the launch template | | [node\_group\_arn](#output\_node\_group\_arn) | Amazon Resource Name (ARN) of the EKS Node Group | | [node\_group\_autoscaling\_group\_names](#output\_node\_group\_autoscaling\_group\_names) | List of the autoscaling group names | | [node\_group\_id](#output\_node\_group\_id) | EKS Cluster name and EKS Node Group name separated by a colon (`:`) | diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index 2899692c6b..9d7535332f 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -17,6 +17,11 @@ output "launch_template_latest_version" { value = try(aws_launch_template.this[0].latest_version, "") } +output "launch_template_name" { + description = "The name of the launch template" + value = try(aws_launch_template.this[0].name, "") +} + ################################################################################ # Node Group ################################################################################ diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 2398fb1d7a..72a5aa723d 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -386,7 +386,7 @@ variable "security_group_name" { variable "security_group_use_name_prefix" { description = "Determines whether the security group name (`security_group_name`) is used as a prefix" - type = string + type = bool default = true } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 43b5eb4ba6..eb0b484443 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -156,7 +156,7 @@ module "self_managed_node_group" { | [security\_group\_name](#input\_security\_group\_name) | Name to use on security group created | `string` | `null` | no | | [security\_group\_rules](#input\_security\_group\_rules) | List of security group rules to add to the security group created | `any` | `{}` | no | | [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the security group created | `map(string)` | `{}` | no | -| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `string` | `true` | no | +| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `bool` | `true` | no | | [service\_linked\_role\_arn](#input\_service\_linked\_role\_arn) | The ARN of the service-linked role that the ASG will use to call other AWS services | `string` | `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones` | `list(string)` | `null` | no | | [suspended\_processes](#input\_suspended\_processes) | A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly | `list(string)` | `[]` | no | @@ -200,6 +200,7 @@ module "self_managed_node_group" { | [launch\_template\_arn](#output\_launch\_template\_arn) | The ARN of the launch template | | [launch\_template\_id](#output\_launch\_template\_id) | The ID of the launch template | | [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | +| [launch\_template\_name](#output\_launch\_template\_name) | The name of the launch template | | [platform](#output\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | | [security\_group\_arn](#output\_security\_group\_arn) | Amazon Resource Name (ARN) of the security group | | [security\_group\_id](#output\_security\_group\_id) | ID of the security group | diff --git a/modules/self-managed-node-group/outputs.tf b/modules/self-managed-node-group/outputs.tf index 552e9703be..0cfba97d25 100644 --- a/modules/self-managed-node-group/outputs.tf +++ b/modules/self-managed-node-group/outputs.tf @@ -17,6 +17,11 @@ output "launch_template_latest_version" { value = try(aws_launch_template.this[0].latest_version, "") } +output "launch_template_name" { + description = "The name of the launch template" + value = try(aws_launch_template.this[0].name, "") +} + ################################################################################ # autoscaling group ################################################################################ diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 5aa20066c5..3734d9068c 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -504,7 +504,7 @@ variable "security_group_name" { variable "security_group_use_name_prefix" { description = "Determines whether the security group name (`security_group_name`) is used as a prefix" - type = string + type = bool default = true } diff --git a/variables.tf b/variables.tf index c5bde1abb5..df2ee51638 100644 --- a/variables.tf +++ b/variables.tf @@ -244,7 +244,7 @@ variable "cluster_security_group_name" { variable "cluster_security_group_use_name_prefix" { description = "Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix" - type = string + type = bool default = true } @@ -300,7 +300,7 @@ variable "node_security_group_name" { variable "node_security_group_use_name_prefix" { description = "Determines whether node security group name (`node_security_group_name`) is used as a prefix" - type = string + type = bool default = true } @@ -426,7 +426,7 @@ variable "iam_role_tags" { variable "cluster_encryption_policy_use_name_prefix" { description = "Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix" - type = string + type = bool default = true } From 0957d674456373a47ab22d09073b4ef7af584fd3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 17 Aug 2022 10:25:11 +0000 Subject: [PATCH 586/962] chore(release): version 18.28.0 [skip ci] ## [18.28.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.27.1...v18.28.0) (2022-08-17) ### Features * Add output for launch template name, and correct variable type value ([#2205](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2205)) ([0a52d69](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0a52d690d54a7c39fd4e0d46db36d200f7ef679e)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ff6863cd4..50a8016b61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.28.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.27.1...v18.28.0) (2022-08-17) + + +### Features + +* Add output for launch template name, and correct variable type value ([#2205](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2205)) ([0a52d69](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0a52d690d54a7c39fd4e0d46db36d200f7ef679e)) + ### [18.27.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.27.0...v18.27.1) (2022-08-09) From f576a6f9ea523c94a7bb5420d5ab3ed8c7d3fec7 Mon Sep 17 00:00:00 2001 From: Karsten Siemer Date: Fri, 26 Aug 2022 21:45:07 +0200 Subject: [PATCH 587/962] feat: Allow TLS provider to use versions 3.0+ (i.e. - `>= 3.0`) (#2211) --- README.md | 4 ++-- examples/eks_managed_node_group/README.md | 4 ++-- examples/eks_managed_node_group/versions.tf | 2 +- examples/self_managed_node_group/README.md | 4 ++-- examples/self_managed_node_group/versions.tf | 2 +- versions.tf | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4ee1015837..ae234a9252 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | -| [tls](#requirement\_tls) | ~> 3.0 | +| [tls](#requirement\_tls) | >= 3.0 | ## Providers @@ -218,7 +218,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|---------| | [aws](#provider\_aws) | >= 3.72 | | [kubernetes](#provider\_kubernetes) | >= 2.10 | -| [tls](#provider\_tls) | ~> 3.0 | +| [tls](#provider\_tls) | >= 3.0 | ## Modules diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index ea51a159cb..654a116037 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -60,14 +60,14 @@ Note that this example may create resources which cost money. Run `terraform des | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | -| [tls](#requirement\_tls) | ~> 3.0 | +| [tls](#requirement\_tls) | >= 3.0 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | -| [tls](#provider\_tls) | ~> 3.0 | +| [tls](#provider\_tls) | >= 3.0 | ## Modules diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index 7fa16dcd48..fde7af0f23 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -8,7 +8,7 @@ terraform { } tls = { source = "hashicorp/tls" - version = "~> 3.0" + version = ">= 3.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index cfdff57cb7..7831b49f73 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -28,14 +28,14 @@ Note that this example may create resources which cost money. Run `terraform des | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | -| [tls](#requirement\_tls) | ~> 3.0 | +| [tls](#requirement\_tls) | >= 3.0 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | -| [tls](#provider\_tls) | ~> 3.0 | +| [tls](#provider\_tls) | >= 3.0 | ## Modules diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index 7fa16dcd48..fde7af0f23 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -8,7 +8,7 @@ terraform { } tls = { source = "hashicorp/tls" - version = "~> 3.0" + version = ">= 3.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/versions.tf b/versions.tf index 7fa16dcd48..fde7af0f23 100644 --- a/versions.tf +++ b/versions.tf @@ -8,7 +8,7 @@ terraform { } tls = { source = "hashicorp/tls" - version = "~> 3.0" + version = ">= 3.0" } kubernetes = { source = "hashicorp/kubernetes" From 4ca9187d53bde3cfb1f3672d9a8e23a8032f1145 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 26 Aug 2022 19:45:43 +0000 Subject: [PATCH 588/962] chore(release): version 18.29.0 [skip ci] ## [18.29.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.28.0...v18.29.0) (2022-08-26) ### Features * Allow TLS provider to use versions 3.0+ (i.e. - `>= 3.0`) ([#2211](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2211)) ([f576a6f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f576a6f9ea523c94a7bb5420d5ab3ed8c7d3fec7)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a8016b61..95f535c1e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.29.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.28.0...v18.29.0) (2022-08-26) + + +### Features + +* Allow TLS provider to use versions 3.0+ (i.e. - `>= 3.0`) ([#2211](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2211)) ([f576a6f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f576a6f9ea523c94a7bb5420d5ab3ed8c7d3fec7)) + ## [18.28.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.27.1...v18.28.0) (2022-08-17) From 9e184dddf4daabdd4f62bf938b03894009aa4de4 Mon Sep 17 00:00:00 2001 From: Thomas Colomb Date: Wed, 7 Sep 2022 16:33:26 +0200 Subject: [PATCH 589/962] chore: Correct example ASG tag(s) to match `cluster-autoscaler` requirements (#2220) --- examples/eks_managed_node_group/main.tf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index ef702dff87..12db52be62 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -675,6 +675,14 @@ data "aws_ami" "eks_default_bottlerocket" { ################################################################################ locals { + + # We need to lookup K8s taint effect from the AWS API value + taint_effects = { + NO_SCHEDULE = "NoSchedule" + NO_EXECUTE = "NoExecute" + PREFER_NO_SCHEDULE = "PreferNoSchedule" + } + cluster_autoscaler_label_tags = merge([ for name, group in module.eks.eks_managed_node_groups : { for label_name, label_value in coalesce(group.node_group_labels, {}) : "${name}|label|${label_name}" => { @@ -690,7 +698,7 @@ locals { for taint in coalesce(group.node_group_taints, []) : "${name}|taint|${taint.key}" => { autoscaling_group = group.node_group_autoscaling_group_names[0], key = "k8s.io/cluster-autoscaler/node-template/taint/${taint.key}" - value = "${taint.value}:${taint.effect}" + value = "${taint.value}:${local.taint_effects[taint.effect]}" } } ]...) From 57bb667f20e072bb1a6ca1ffa9beef6052895acb Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 7 Sep 2022 17:52:27 -0400 Subject: [PATCH 590/962] docs: Add section to FAQ about conflicting security group tags (#2225) Co-authored-by: Anton Babenko --- .pre-commit-config.yaml | 2 +- docs/faq.md | 42 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 727e21c06e..6a8a23ae46 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.74.1 + rev: v1.75.0 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/docs/faq.md b/docs/faq.md index e729a3bf6b..4a73b7a8d7 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,11 +1,53 @@ # Frequently Asked Questions +- [I received an error: `expect exactly one securityGroup tagged with kubernetes.io/cluster/ ...`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#i-received-an-error-expect-exactly-one-securitygroup-tagged-with-kubernetesioclustername-) - [I received an error: `Error: Invalid for_each argument ...`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#i-received-an-error-error-invalid-for_each-argument-) - [Why are nodes not being registered?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-nodes-not-being-registered) - [Why are there no changes when a node group's `desired_size` is modified?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-there-no-changes-when-a-node-groups-desired_size-is-modified) - [How can I deploy Windows based nodes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-can-i-deploy-windows-based-nodes) - [How do I access compute resource attributes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-do-i-access-compute-resource-attributes) +### I received an error: `expect exactly one securityGroup tagged with kubernetes.io/cluster/ ...` + +By default, EKS creates a cluster primary security group that is created outside of the module and the EKS service adds the tag `{ "kubernetes.io/cluster/" = "owned" }`. This on its own does not cause any conflicts for addons such as the AWS Load Balancer Controller until users decide to attach both the cluster primary security group and the shared node security group created by the module (by setting `attach_cluster_primary_security_group = true`). The issue is not with having multiple security groups in your account with this tag key:value combination, but having multiple security groups with this tag key:value combination attached to nodes in the same cluster. There are a few ways to resolve this depending on your use case/intentions: + +⚠️ `` below needs to be replaced with the name of your cluster + +1. If you want to use the cluster primary security group, you can disable the creation of the shared node security group with: + +```hcl + create_node_security_group = false # default is true + attach_cluster_primary_security_group = true # default is false +``` + +2. If you want to use the cluster primary security group, you can disable the tag passed to the node security group by overriding the tag expected value like: + +```hcl + attach_cluster_primary_security_group = true # default is false + + node_security_group_tags = { + "kubernetes.io/cluster/" = "" # or any other value other than "owned" + } +``` + +3. By overriding the tag expected value on the cluster primary security group like: + +```hcl + attach_cluster_primary_security_group = true # default is false + + cluster_tags = { + "kubernetes.io/cluster/" = "" # or any other value other than "owned" + } +``` + +4. By not attaching the cluster primary security group. The cluster primary security group has quite broad access and the module has instead provided a security group with the minimum amount of access to launch an empty EKS cluster successfully and users are encouraged to open up access when necessary to support their workload. + +```hcl + attach_cluster_primary_security_group = false # this is the default for the module +``` + +In theory, if you are attaching the cluster primary security group, you shouldn't need to use the shared node security group created by the module. However, this is left up to users to decide for their requirements and use case. + ### I received an error: `Error: Invalid for_each argument ...` Users may encounter an error such as `Error: Invalid for_each argument - The "for_each" value depends on resource attributes that cannot be determined until apply, so Terraform cannot predict how many instances will be created. To work around this, use the -target argument to first apply ...` From c5944e5fb6ea07429ef79f5fe5592e7111567e1e Mon Sep 17 00:00:00 2001 From: Charles Bushong Date: Mon, 26 Sep 2022 10:02:13 -0400 Subject: [PATCH 591/962] fix: Set `image_id` to come from the launch template instead of data source for self-managed node groups (#2239) --- modules/self-managed-node-group/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/self-managed-node-group/outputs.tf b/modules/self-managed-node-group/outputs.tf index 0cfba97d25..e9f52db14f 100644 --- a/modules/self-managed-node-group/outputs.tf +++ b/modules/self-managed-node-group/outputs.tf @@ -153,7 +153,7 @@ output "platform" { output "image_id" { description = "ID of the image" - value = try(data.aws_ami.eks_default[0].image_id, "") + value = try(aws_launch_template.this[0].image_id, "") } output "user_data" { From 32000068258828b812b3b6f76efcb2b452b810f3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 26 Sep 2022 14:02:57 +0000 Subject: [PATCH 592/962] chore(release): version 18.29.1 [skip ci] ### [18.29.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.29.0...v18.29.1) (2022-09-26) ### Bug Fixes * Set `image_id` to come from the launch template instead of data source for self-managed node groups ([#2239](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2239)) ([c5944e5](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c5944e5fb6ea07429ef79f5fe5592e7111567e1e)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95f535c1e9..ada874bc23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.29.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.29.0...v18.29.1) (2022-09-26) + + +### Bug Fixes + +* Set `image_id` to come from the launch template instead of data source for self-managed node groups ([#2239](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2239)) ([c5944e5](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c5944e5fb6ea07429ef79f5fe5592e7111567e1e)) + ## [18.29.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.28.0...v18.29.0) (2022-08-26) From a74e98017b5dc7ed396cf26bfaf98ff7951c9e2e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 29 Sep 2022 15:34:20 -0400 Subject: [PATCH 593/962] feat: Add output for cluster TLS certificate SHA1 fingerprint and provider tags to cluster primary security group (#2249) --- README.md | 2 ++ examples/complete/README.md | 1 + examples/complete/outputs.tf | 5 +++++ examples/eks_managed_node_group/README.md | 1 + examples/eks_managed_node_group/outputs.tf | 5 +++++ examples/fargate_profile/README.md | 1 + examples/fargate_profile/outputs.tf | 5 +++++ examples/karpenter/README.md | 1 + examples/karpenter/outputs.tf | 5 +++++ examples/self_managed_node_group/README.md | 1 + examples/self_managed_node_group/outputs.tf | 5 +++++ main.tf | 6 +++++- modules/eks-managed-node-group/README.md | 2 +- modules/eks-managed-node-group/variables.tf | 2 +- outputs.tf | 5 +++++ 15 files changed, 44 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ae234a9252..c82ffd5fe0 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | | [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_default_tags.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -370,6 +371,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [cluster\_version](#output\_cluster\_version) | The Kubernetes version for the cluster | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | diff --git a/examples/complete/README.md b/examples/complete/README.md index 7aa52f0740..6961891591 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -88,6 +88,7 @@ No inputs. | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index c612b0fe3d..c6b06de812 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -84,6 +84,11 @@ output "oidc_provider_arn" { value = module.eks.oidc_provider_arn } +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = module.eks.cluster_tls_certificate_sha1_fingerprint +} + ################################################################################ # IAM Role ################################################################################ diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 654a116037..9014bc660b 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -123,6 +123,7 @@ No inputs. | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | diff --git a/examples/eks_managed_node_group/outputs.tf b/examples/eks_managed_node_group/outputs.tf index 3e9e8dd23e..6e31908292 100644 --- a/examples/eks_managed_node_group/outputs.tf +++ b/examples/eks_managed_node_group/outputs.tf @@ -103,6 +103,11 @@ output "oidc_provider_arn" { value = module.eks.oidc_provider_arn } +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = module.eks.cluster_tls_certificate_sha1_fingerprint +} + ################################################################################ # IAM Role ################################################################################ diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 81f0366692..d8a1bfecf3 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -68,6 +68,7 @@ No inputs. | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | diff --git a/examples/fargate_profile/outputs.tf b/examples/fargate_profile/outputs.tf index 3e9e8dd23e..6e31908292 100644 --- a/examples/fargate_profile/outputs.tf +++ b/examples/fargate_profile/outputs.tf @@ -103,6 +103,11 @@ output "oidc_provider_arn" { value = module.eks.oidc_provider_arn } +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = module.eks.cluster_tls_certificate_sha1_fingerprint +} + ################################################################################ # IAM Role ################################################################################ diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index fa06e90170..761299aa54 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -108,6 +108,7 @@ No inputs. | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | diff --git a/examples/karpenter/outputs.tf b/examples/karpenter/outputs.tf index bfe53985b1..dbbec2334e 100644 --- a/examples/karpenter/outputs.tf +++ b/examples/karpenter/outputs.tf @@ -84,6 +84,11 @@ output "oidc_provider_arn" { value = module.eks.oidc_provider_arn } +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = module.eks.cluster_tls_certificate_sha1_fingerprint +} + ################################################################################ # IAM Role ################################################################################ diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 7831b49f73..a543d6454b 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -85,6 +85,7 @@ No inputs. | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | diff --git a/examples/self_managed_node_group/outputs.tf b/examples/self_managed_node_group/outputs.tf index 3e9e8dd23e..6e31908292 100644 --- a/examples/self_managed_node_group/outputs.tf +++ b/examples/self_managed_node_group/outputs.tf @@ -103,6 +103,11 @@ output "oidc_provider_arn" { value = module.eks.oidc_provider_arn } +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = module.eks.cluster_tls_certificate_sha1_fingerprint +} + ################################################################################ # IAM Role ################################################################################ diff --git a/main.tf b/main.tf index 6b0dc66ecc..011a26c396 100644 --- a/main.tf +++ b/main.tf @@ -1,5 +1,6 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} +data "aws_default_tags" "current" {} locals { create = var.create && var.putin_khuylo @@ -66,7 +67,10 @@ resource "aws_ec2_tag" "cluster_primary_security_group" { # This should not affect the name of the cluster primary security group # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2006 # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2008 - for_each = { for k, v in merge(var.tags, var.cluster_tags) : k => v if local.create && k != "Name" && var.create_cluster_primary_security_group_tags } + # `aws_default_tags` is merged in to "dedupe" tags and stabilize tag updates + for_each = { for k, v in merge(var.tags, var.cluster_tags, data.aws_default_tags.current.tags) : + k => v if local.create && k != "Name" && var.create_cluster_primary_security_group_tags + } resource_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id key = each.key diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 44aa062d4c..eb8f35dbd3 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -113,7 +113,7 @@ module "eks_managed_node_group" { | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance(s) will be EBS-optimized | `bool` | `null` | no | | [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `map(string)` | `{}` | no | | [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | -| [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template | `bool` | `false` | no | +| [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template. Only valid when using a custom AMI via `ami_id` | `bool` | `false` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | | [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | | [force\_update\_version](#input\_force\_update\_version) | Force version update if existing pods are unable to be drained due to a pod disruption budget issue | `bool` | `null` | no | diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 72a5aa723d..0000827719 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -21,7 +21,7 @@ variable "platform" { ################################################################################ variable "enable_bootstrap_user_data" { - description = "Determines whether the bootstrap configurations are populated within the user data template" + description = "Determines whether the bootstrap configurations are populated within the user data template. Only valid when using a custom AMI via `ami_id`" type = bool default = false } diff --git a/outputs.tf b/outputs.tf index 25b8a0b24a..7fb6d8ce98 100644 --- a/outputs.tf +++ b/outputs.tf @@ -108,6 +108,11 @@ output "oidc_provider_arn" { value = try(aws_iam_openid_connect_provider.oidc_provider[0].arn, "") } +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = try(data.tls_certificate.this[0].certificates[0].sha1_fingerprint, "") +} + ################################################################################ # IAM Role ################################################################################ From 7f901840c430c0b631cc4066a87779b915a07524 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 29 Sep 2022 19:34:55 +0000 Subject: [PATCH 594/962] chore(release): version 18.30.0 [skip ci] ## [18.30.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.29.1...v18.30.0) (2022-09-29) ### Features * Add output for cluster TLS certificate SHA1 fingerprint and provider tags to cluster primary security group ([#2249](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2249)) ([a74e980](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a74e98017b5dc7ed396cf26bfaf98ff7951c9e2e)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ada874bc23..b1115fe039 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.30.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.29.1...v18.30.0) (2022-09-29) + + +### Features + +* Add output for cluster TLS certificate SHA1 fingerprint and provider tags to cluster primary security group ([#2249](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2249)) ([a74e980](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a74e98017b5dc7ed396cf26bfaf98ff7951c9e2e)) + ### [18.29.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.29.0...v18.29.1) (2022-09-26) From ac4d549629aa64bbd92f80486bef904a9098e0fa Mon Sep 17 00:00:00 2001 From: Zvika Nadav Date: Tue, 11 Oct 2022 15:28:06 -0500 Subject: [PATCH 595/962] fix: Update CloudWatch log group creation deny policy to use wildcard (#2267) Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 2 +- main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6a8a23ae46..74f3751c74 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.75.0 + rev: v1.76.0 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/main.tf b/main.tf index 011a26c396..6de17d5cf1 100644 --- a/main.tf +++ b/main.tf @@ -279,7 +279,7 @@ resource "aws_iam_role" "this" { { Action = ["logs:CreateLogGroup"] Effect = "Deny" - Resource = aws_cloudwatch_log_group.this[0].arn + Resource = "*" }, ] }) From 8dc5ad447884883ded23e3fed149ea87337dfafd Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 11 Oct 2022 20:28:33 +0000 Subject: [PATCH 596/962] chore(release): version 18.30.1 [skip ci] ### [18.30.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.0...v18.30.1) (2022-10-11) ### Bug Fixes * Update CloudWatch log group creation deny policy to use wildcard ([#2267](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2267)) ([ac4d549](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ac4d549629aa64bbd92f80486bef904a9098e0fa)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1115fe039..54e0ac0556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.30.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.0...v18.30.1) (2022-10-11) + + +### Bug Fixes + +* Update CloudWatch log group creation deny policy to use wildcard ([#2267](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2267)) ([ac4d549](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ac4d549629aa64bbd92f80486bef904a9098e0fa)) + ## [18.30.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.29.1...v18.30.0) (2022-09-29) From 28ccecefe22d81a3a7febbbc3efc17c6590f88e1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 14 Oct 2022 09:16:57 -0400 Subject: [PATCH 597/962] fix: Disable creation of cluster security group rules that map to node security group when `create_node_security_group` = `false` (#2274) * fix: Disable creation of cluster security group rules that map to node security group when `create_node_security_group` = `false` * feat: Update Fargate example to run only Fargate and show disabling of both cluster and node security groups * fix: Ensure CoreDNS changes are made ahead of install --- docs/faq.md | 4 +- examples/fargate_profile/README.md | 10 +- examples/fargate_profile/main.tf | 195 ++++++++++++++++++++------- examples/fargate_profile/versions.tf | 10 +- main.tf | 5 +- 5 files changed, 171 insertions(+), 53 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 4a73b7a8d7..c1ab564ee0 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -26,7 +26,7 @@ By default, EKS creates a cluster primary security group that is created outside attach_cluster_primary_security_group = true # default is false node_security_group_tags = { - "kubernetes.io/cluster/" = "" # or any other value other than "owned" + "kubernetes.io/cluster/" = null # or any other value other than "owned" } ``` @@ -36,7 +36,7 @@ By default, EKS creates a cluster primary security group that is created outside attach_cluster_primary_security_group = true # default is false cluster_tags = { - "kubernetes.io/cluster/" = "" # or any other value other than "owned" + "kubernetes.io/cluster/" = null # or any other value other than "owned" } ``` diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index d8a1bfecf3..dddbc5755e 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -21,13 +21,16 @@ Note that this example may create resources which cost money. Run `terraform des |------|---------| | [terraform](#requirement\_terraform) | >= 0.13.1 | | [aws](#requirement\_aws) | >= 3.72 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | +| [helm](#requirement\_helm) | >= 2.7 | +| [null](#requirement\_null) | >= 3.0 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 3.72 | +| [helm](#provider\_helm) | >= 2.7 | +| [null](#provider\_null) | >= 3.0 | ## Modules @@ -41,6 +44,11 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| | [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [helm_release.coredns](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [null_resource.modify_kube_dns](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.remove_default_coredns_deployment](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | +| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | ## Inputs diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index 6c80e22c41..a7b3f8f903 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -2,6 +2,20 @@ provider "aws" { region = local.region } +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + } + } +} + locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" cluster_version = "1.22" @@ -27,14 +41,8 @@ module "eks" { cluster_endpoint_public_access = true cluster_addons = { - # Note: https://docs.aws.amazon.com/eks/latest/userguide/fargate-getting-started.html#fargate-gs-coredns - coredns = { - resolve_conflicts = "OVERWRITE" - } kube-proxy = {} - vpc-cni = { - resolve_conflicts = "OVERWRITE" - } + vpc-cni = {} } cluster_encryption_config = [{ @@ -45,28 +53,13 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets - # You require a node group to schedule coredns which is critical for running correctly internal DNS. - # If you want to use only fargate you must follow docs `(Optional) Update CoreDNS` - # available under https://docs.aws.amazon.com/eks/latest/userguide/fargate-getting-started.html - eks_managed_node_groups = { - example = { - desired_size = 1 - - instance_types = ["t3.large"] - labels = { - Example = "managed_node_groups" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - tags = { - ExtraTag = "example" - } - } - } + # Fargate profiles use the cluster primary security group so these are not utilized + create_cluster_security_group = false + create_node_security_group = false fargate_profiles = { - default = { - name = "default" + example = { + name = "example" selectors = [ { namespace = "backend" @@ -75,15 +68,18 @@ module "eks" { } }, { - namespace = "default" + namespace = "app-*" labels = { - WorkerType = "fargate" + Application = "app-wildcard" } } ] + # Using specific subnets instead of the subnets supplied for the cluster itself + subnet_ids = [module.vpc.private_subnets[1]] + tags = { - Owner = "default" + Owner = "secondary" } timeouts = { @@ -92,29 +88,138 @@ module "eks" { } } - secondary = { - name = "secondary" + kube_system = { + name = "kube-system" selectors = [ - { - namespace = "default" - labels = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - } + { namespace = "kube-system" } ] + } + } - # Using specific subnets instead of the subnets supplied for the cluster itself - subnet_ids = [module.vpc.private_subnets[1]] + tags = local.tags +} - tags = { - Owner = "secondary" +################################################################################ +# Modify EKS CoreDNS Deployment +################################################################################ + +data "aws_eks_cluster_auth" "this" { + name = module.eks.cluster_id +} + +locals { + kubeconfig = yamlencode({ + apiVersion = "v1" + kind = "Config" + current-context = "terraform" + clusters = [{ + name = module.eks.cluster_id + cluster = { + certificate-authority-data = module.eks.cluster_certificate_authority_data + server = module.eks.cluster_endpoint + } + }] + contexts = [{ + name = "terraform" + context = { + cluster = module.eks.cluster_id + user = "terraform" } + }] + users = [{ + name = "terraform" + user = { + token = data.aws_eks_cluster_auth.this.token + } + }] + }) +} + +# Separate resource so that this is only ever executed once +resource "null_resource" "remove_default_coredns_deployment" { + triggers = {} + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = base64encode(local.kubeconfig) } + + # We are removing the deployment provided by the EKS service and replacing it through the self-managed CoreDNS Helm addon + # However, we are maintaing the existing kube-dns service and annotating it for Helm to assume control + command = <<-EOT + kubectl --namespace kube-system delete deployment coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) + EOT } +} - tags = local.tags +resource "null_resource" "modify_kube_dns" { + triggers = {} + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = base64encode(local.kubeconfig) + } + + # We are maintaing the existing kube-dns service and annotating it for Helm to assume control + command = <<-EOT + echo "Setting implicit dependency on ${module.eks.fargate_profiles["kube_system"].fargate_profile_pod_execution_role_arn}" + kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-name=coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) + kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-namespace=kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) + kubectl --namespace kube-system label --overwrite service kube-dns app.kubernetes.io/managed-by=Helm --kubeconfig <(echo $KUBECONFIG | base64 --decode) + EOT + } + + depends_on = [ + null_resource.remove_default_coredns_deployment + ] +} + +################################################################################ +# CoreDNS Helm Chart (self-managed) +################################################################################ + +data "aws_eks_addon_version" "this" { + for_each = toset(["coredns"]) + + addon_name = each.value + kubernetes_version = module.eks.cluster_version + most_recent = true +} + +resource "helm_release" "coredns" { + name = "coredns" + namespace = "kube-system" + create_namespace = false + description = "CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS Services" + chart = "coredns" + version = "1.19.4" + repository = "https://coredns.github.io/helm" + + # For EKS image repositories https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html + values = [ + <<-EOT + image: + repository: 602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/coredns + tag: ${data.aws_eks_addon_version.this["coredns"].version} + deployment: + name: coredns + annotations: + eks.amazonaws.com/compute-type: fargate + service: + name: kube-dns + annotations: + eks.amazonaws.com/compute-type: fargate + podAnnotations: + eks.amazonaws.com/compute-type: fargate + EOT + ] + + depends_on = [ + # Need to ensure the CoreDNS updates are peformed before provisioning + null_resource.modify_kube_dns + ] } ################################################################################ diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index 6d6dc45be6..5128bc99e3 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -6,9 +6,13 @@ terraform { source = "hashicorp/aws" version = ">= 3.72" } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" + helm = { + source = "hashicorp/helm" + version = ">= 2.7" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" } } } diff --git a/main.tf b/main.tf index 6de17d5cf1..a17d4a1499 100644 --- a/main.tf +++ b/main.tf @@ -128,7 +128,8 @@ locals { cluster_security_group_id = local.create_cluster_sg ? aws_security_group.cluster[0].id : var.cluster_security_group_id - cluster_security_group_rules = { + # Do not add rules to node security group if the module is not creating it + cluster_security_group_rules = local.create_node_sg ? { ingress_nodes_443 = { description = "Node groups to cluster API" protocol = "tcp" @@ -153,7 +154,7 @@ locals { type = "egress" source_node_security_group = true } - } + } : {} } resource "aws_security_group" "cluster" { From cc3fe0590b6d7c8bf2a1690c5331fa92527efbe4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 14 Oct 2022 13:17:33 +0000 Subject: [PATCH 598/962] chore(release): version 18.30.2 [skip ci] ### [18.30.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.1...v18.30.2) (2022-10-14) ### Bug Fixes * Disable creation of cluster security group rules that map to node security group when `create_node_security_group` = `false` ([#2274](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2274)) ([28ccece](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/28ccecefe22d81a3a7febbbc3efc17c6590f88e1)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54e0ac0556..b059c79a23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.30.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.1...v18.30.2) (2022-10-14) + + +### Bug Fixes + +* Disable creation of cluster security group rules that map to node security group when `create_node_security_group` = `false` ([#2274](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2274)) ([28ccece](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/28ccecefe22d81a3a7febbbc3efc17c6590f88e1)) + ### [18.30.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.0...v18.30.1) (2022-10-11) From 364c60d572e85676adca8f6e62679de7d9551271 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 7 Nov 2022 14:17:59 -0500 Subject: [PATCH 599/962] fix: Update CI configuration files to use latest version (#2293) --- .github/workflows/lock.yml | 21 +++++++++++++++++++++ .github/workflows/pr-title.yml | 2 +- .github/workflows/pre-commit.yml | 18 +++++++++--------- .github/workflows/release.yml | 2 +- .github/workflows/stale-actions.yaml | 2 +- 5 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/lock.yml diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 0000000000..d887a660fe --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,21 @@ +name: 'Lock Threads' + +on: + schedule: + - cron: '50 1 * * *' + +jobs: + lock: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + issue-comment: > + I'm going to lock this issue because it has been closed for _30 days_ ⏳. This helps our maintainers find and focus on the active issues. + If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further. + issue-inactive-days: '30' + pr-comment: > + I'm going to lock this pull request because it has been closed for _30 days_ ⏳. This helps our maintainers find and focus on the active issues. + If you have found a problem that seems related to this change, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further. + pr-inactive-days: '30' diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 168011c485..cb32a0f815 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -14,7 +14,7 @@ jobs: steps: # Please look up the latest version from # https://github.com/amannn/action-semantic-pull-request/releases - - uses: amannn/action-semantic-pull-request@v3.4.6 + - uses: amannn/action-semantic-pull-request@v5.0.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index b8f1b8a5ab..06270c8a90 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -17,11 +17,11 @@ jobs: directories: ${{ steps.dirs.outputs.directories }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Get root directories id: dirs - uses: clowdhaus/terraform-composite-actions/directories@v1.3.0 + uses: clowdhaus/terraform-composite-actions/directories@v1.8.0 preCommitMinVersions: name: Min TF pre-commit @@ -32,18 +32,18 @@ jobs: directory: ${{ fromJson(needs.collectInputs.outputs.directories) }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.0.3 + uses: clowdhaus/terraform-min-max@v1.2.0 with: directory: ${{ matrix.directory }} - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory != '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.3.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} args: 'terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/*' @@ -51,7 +51,7 @@ jobs: - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory == '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.3.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} args: 'terraform_validate --color=always --show-diff-on-failure --files $(ls *.tf)' @@ -62,17 +62,17 @@ jobs: needs: collectInputs steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.ref }} repository: ${{github.event.pull_request.head.repo.full_name}} - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.0.3 + uses: clowdhaus/terraform-min-max@v1.2.0 - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.3.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0 with: terraform-version: ${{ steps.minMax.outputs.maxVersion }} terraform-docs-version: ${{ env.TERRAFORM_DOCS_VERSION }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e8a26ca6c0..98c8b25851 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: if: github.repository_owner == 'terraform-aws-modules' steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false fetch-depth: 0 diff --git a/.github/workflows/stale-actions.yaml b/.github/workflows/stale-actions.yaml index c09ae1d58f..50379957fd 100644 --- a/.github/workflows/stale-actions.yaml +++ b/.github/workflows/stale-actions.yaml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v4 + - uses: actions/stale@v6 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Staling issues and PR's From 417788cc2b4cfa737540ad3be9a39be765a5e968 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 7 Nov 2022 19:18:35 +0000 Subject: [PATCH 600/962] chore(release): version 18.30.3 [skip ci] ### [18.30.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.2...v18.30.3) (2022-11-07) ### Bug Fixes * Update CI configuration files to use latest version ([#2293](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2293)) ([364c60d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/364c60d572e85676adca8f6e62679de7d9551271)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b059c79a23..774a3bc7f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.30.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.2...v18.30.3) (2022-11-07) + + +### Bug Fixes + +* Update CI configuration files to use latest version ([#2293](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2293)) ([364c60d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/364c60d572e85676adca8f6e62679de7d9551271)) + ### [18.30.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.1...v18.30.2) (2022-10-14) From 1bc86e19aa0a8776e578d4f58b852bffa28e2aac Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 8 Nov 2022 03:29:42 -0500 Subject: [PATCH 601/962] chore: Update examples to better demonstrate questions raised through issues (#2294) Resolves undefined --- examples/complete/main.tf | 62 ++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 112e3b531d..a34761e687 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -1,11 +1,5 @@ provider "aws" { region = local.region - - default_tags { - tags = { - ExampleDefaultTag = "ExampleDefaultValue" - } - } } provider "kubernetes" { @@ -196,12 +190,14 @@ module "eks" { } } - # OIDC Identity provider - cluster_identity_providers = { - sts = { - client_id = "sts.amazonaws.com" - } - } + # Create a new cluster where both an identity provider and Fargate profile is created + # will result in conflicts since only one can take place at a time + # # OIDC Identity provider + # cluster_identity_providers = { + # sts = { + # client_id = "sts.amazonaws.com" + # } + # } # aws-auth configmap manage_aws_auth_configmap = true @@ -216,10 +212,30 @@ module "eks" { aws_auth_roles = [ { - rolearn = "arn:aws:iam::66666666666:role/role1" - username = "role1" - groups = ["system:masters"] + rolearn = module.eks_managed_node_group.iam_role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] }, + { + rolearn = module.self_managed_node_group.iam_role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, + { + rolearn = module.fargate_profile.fargate_profile_pod_execution_role_arn + username = "system:node:{{SessionName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + "system:node-proxier", + ] + } ] aws_auth_users = [ @@ -261,6 +277,20 @@ module "eks_managed_node_group" { module.eks.cluster_security_group_id, ] + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + + # this will get added to what AWS provides + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + + [settings.kubernetes.node-labels] + "label1" = "foo" + "label2" = "bar" + EOT + tags = merge(local.tags, { Separate = "eks-managed-node-group" }) } @@ -282,8 +312,6 @@ module "self_managed_node_group" { module.eks.cluster_security_group_id, ] - use_default_tags = true - tags = merge(local.tags, { Separate = "self-managed-node-group" }) } From f24de3326d3c12ce61fbaefe1e3dbe7418d8bc85 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 21 Nov 2022 13:50:34 -0500 Subject: [PATCH 602/962] feat: New Karpenter sub-module for easily enabling Karpenter on EKS (#2303) --- README.md | 8 +- docs/compute_resources.md | 6 +- docs/irsa_integration.md | 2 +- examples/complete/README.md | 3 +- examples/complete/main.tf | 8 +- examples/complete/outputs.tf | 7 +- examples/eks_managed_node_group/README.md | 3 +- examples/eks_managed_node_group/main.tf | 4 +- examples/eks_managed_node_group/outputs.tf | 7 +- examples/fargate_profile/README.md | 3 +- examples/fargate_profile/main.tf | 10 +- examples/fargate_profile/outputs.tf | 7 +- examples/karpenter/README.md | 32 +- examples/karpenter/main.tf | 383 +++++++++++++------- examples/karpenter/outputs.tf | 97 ++++- examples/karpenter/versions.tf | 8 + examples/self_managed_node_group/README.md | 3 +- examples/self_managed_node_group/main.tf | 4 +- examples/self_managed_node_group/outputs.tf | 7 +- modules/eks-managed-node-group/README.md | 2 +- modules/karpenter/README.md | 195 ++++++++++ modules/karpenter/main.tf | 359 ++++++++++++++++++ modules/karpenter/outputs.tf | 89 +++++ modules/karpenter/variables.tf | 226 ++++++++++++ modules/karpenter/versions.tf | 10 + modules/self-managed-node-group/README.md | 2 +- outputs.tf | 7 +- variables.tf | 2 +- 28 files changed, 1334 insertions(+), 160 deletions(-) create mode 100644 modules/karpenter/README.md create mode 100644 modules/karpenter/main.tf create mode 100644 modules/karpenter/outputs.tf create mode 100644 modules/karpenter/variables.tf create mode 100644 modules/karpenter/versions.tf diff --git a/README.md b/README.md index c82ffd5fe0..d003e14724 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Please note that we strive to provide a comprehensive suite of documentation for - [EKS Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) - [Self Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) - [Fargate Profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) +- Support for creating Karpenter related AWS infrastruture resources (e.g. IAM roles, SQS queue, EventBridge rules, etc.) - Support for custom AMI, custom launch template, and custom user data including custom user data template - Support for Amazon Linux 2 EKS Optimized AMI and Bottlerocket nodes - Windows based node support is limited to a default user data template that is provided due to the lack of Windows support and manual steps required to provision Windows based EKS nodes @@ -60,7 +61,7 @@ module "eks" { version = "~> 18.0" cluster_name = "my-cluster" - cluster_version = "1.22" + cluster_version = "1.24" cluster_endpoint_private_access = true cluster_endpoint_public_access = true @@ -295,7 +296,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.22`) | `string` | `null` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.24`) | `string` | `null` | no | | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | @@ -363,8 +364,9 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 51d41025bb..d90cf5f0a8 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -141,9 +141,9 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 1. The `self-managed-node-group` uses the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version by default: ```hcl - cluster_version = "1.22" + cluster_version = "1.24" - # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.22 + # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.24 self_managed_node_groups = { default = {} } @@ -152,7 +152,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 2. To use Bottlerocket, specify the `platform` as `bottlerocket` and supply a Bottlerocket OS AMI: ```hcl - cluster_version = "1.22" + cluster_version = "1.24" self_managed_node_groups = { bottlerocket = { diff --git a/docs/irsa_integration.md b/docs/irsa_integration.md index 6c78bd9575..cde709fd2c 100644 --- a/docs/irsa_integration.md +++ b/docs/irsa_integration.md @@ -8,7 +8,7 @@ module "eks" { source = "terraform-aws-modules/eks/aws" cluster_name = "example" - cluster_version = "1.22" + cluster_version = "1.24" cluster_addons = { vpc-cni = { diff --git a/examples/complete/README.md b/examples/complete/README.md index 6961891591..66f3e8fdf1 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -81,8 +81,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index a34761e687..04817fa78b 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -10,7 +10,7 @@ provider "kubernetes" { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] } } @@ -267,7 +267,7 @@ module "eks_managed_node_group" { source = "../../modules/eks-managed-node-group" name = "separate-eks-mng" - cluster_name = module.eks.cluster_id + cluster_name = module.eks.cluster_name cluster_version = module.eks.cluster_version vpc_id = module.vpc.vpc_id @@ -298,7 +298,7 @@ module "self_managed_node_group" { source = "../../modules/self-managed-node-group" name = "separate-self-mng" - cluster_name = module.eks.cluster_id + cluster_name = module.eks.cluster_name cluster_version = module.eks.cluster_version cluster_endpoint = module.eks.cluster_endpoint cluster_auth_base64 = module.eks.cluster_certificate_authority_data @@ -319,7 +319,7 @@ module "fargate_profile" { source = "../../modules/fargate-profile" name = "separate-fargate-profile" - cluster_name = module.eks.cluster_id + cluster_name = module.eks.cluster_name subnet_ids = module.vpc.private_subnets selectors = [{ diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index c6b06de812..f94c629541 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_name +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = module.eks.cluster_id } diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 9014bc660b..449777a52a 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -115,8 +115,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 12db52be62..050a508810 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -10,13 +10,13 @@ provider "kubernetes" { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] } } locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.22" + cluster_version = "1.24" region = "eu-west-1" tags = { diff --git a/examples/eks_managed_node_group/outputs.tf b/examples/eks_managed_node_group/outputs.tf index 6e31908292..67de532779 100644 --- a/examples/eks_managed_node_group/outputs.tf +++ b/examples/eks_managed_node_group/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_name +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = module.eks.cluster_id } diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index dddbc5755e..2f9cf7da1b 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -68,8 +68,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index a7b3f8f903..0bf15a7442 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -11,14 +11,14 @@ provider "helm" { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] } } } locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.22" + cluster_version = "1.24" region = "eu-west-1" tags = { @@ -104,7 +104,7 @@ module "eks" { ################################################################################ data "aws_eks_cluster_auth" "this" { - name = module.eks.cluster_id + name = module.eks.cluster_name } locals { @@ -113,7 +113,7 @@ locals { kind = "Config" current-context = "terraform" clusters = [{ - name = module.eks.cluster_id + name = module.eks.cluster_name cluster = { certificate-authority-data = module.eks.cluster_certificate_authority_data server = module.eks.cluster_endpoint @@ -122,7 +122,7 @@ locals { contexts = [{ name = "terraform" context = { - cluster = module.eks.cluster_id + cluster = module.eks.cluster_name user = "terraform" } }] diff --git a/examples/fargate_profile/outputs.tf b/examples/fargate_profile/outputs.tf index 6e31908292..67de532779 100644 --- a/examples/fargate_profile/outputs.tf +++ b/examples/fargate_profile/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_name +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = module.eks.cluster_id } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 761299aa54..7b865797a4 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -1,6 +1,6 @@ # Karpenter Example -Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. +Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is running on EKS Fargate yet Karpenter is providing compute in the form of EC2 instances. ## Usage @@ -55,6 +55,8 @@ Note that this example may create resources which cost money. Run `terraform des | [aws](#requirement\_aws) | >= 3.72 | | [helm](#requirement\_helm) | >= 2.4 | | [kubectl](#requirement\_kubectl) | >= 1.14 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | +| [null](#requirement\_null) | >= 3.0 | ## Providers @@ -63,24 +65,29 @@ Note that this example may create resources which cost money. Run `terraform des | [aws](#provider\_aws) | >= 3.72 | | [helm](#provider\_helm) | >= 2.4 | | [kubectl](#provider\_kubectl) | >= 1.14 | +| [null](#provider\_null) | >= 3.0 | ## Modules | Name | Source | Version | |------|--------|---------| | [eks](#module\_eks) | ../.. | n/a | -| [karpenter\_irsa](#module\_karpenter\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.21.1 | +| [karpenter](#module\_karpenter) | ../../modules/karpenter | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources | Name | Type | |------|------| -| [aws_iam_instance_profile.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [helm_release.coredns](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_node_template](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [null_resource.modify_kube_dns](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.remove_default_coredns_deployment](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | +| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | ## Inputs @@ -100,8 +107,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | @@ -112,6 +120,20 @@ No inputs. | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [karpenter\_event\_rules](#output\_karpenter\_event\_rules) | Map of the event rules created and their attributes | +| [karpenter\_instance\_profile\_arn](#output\_karpenter\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | +| [karpenter\_instance\_profile\_id](#output\_karpenter\_instance\_profile\_id) | Instance profile's ID | +| [karpenter\_instance\_profile\_name](#output\_karpenter\_instance\_profile\_name) | Name of the instance profile | +| [karpenter\_instance\_profile\_unique](#output\_karpenter\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | +| [karpenter\_irsa\_arn](#output\_karpenter\_irsa\_arn) | The Amazon Resource Name (ARN) specifying the IAM role for service accounts | +| [karpenter\_irsa\_name](#output\_karpenter\_irsa\_name) | The name of the IAM role for service accounts | +| [karpenter\_irsa\_unique\_id](#output\_karpenter\_irsa\_unique\_id) | Stable and unique string identifying the IAM role for service accounts | +| [karpenter\_queue\_arn](#output\_karpenter\_queue\_arn) | The ARN of the SQS queue | +| [karpenter\_queue\_name](#output\_karpenter\_queue\_name) | The name of the created Amazon SQS queue | +| [karpenter\_queue\_url](#output\_karpenter\_queue\_url) | The URL for the created Amazon SQS queue | +| [karpenter\_role\_arn](#output\_karpenter\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [karpenter\_role\_name](#output\_karpenter\_role\_name) | The name of the IAM role | +| [karpenter\_role\_unique\_id](#output\_karpenter\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 3f43d80b2a..3634132ee2 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -2,13 +2,50 @@ provider "aws" { region = local.region } -data "aws_partition" "current" {} +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } + } +} + +provider "kubectl" { + apply_retry_count = 5 + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + load_config_file = false + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } +} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.22" + cluster_version = "1.24" region = "eu-west-1" - partition = data.aws_partition.current.partition tags = { Example = local.name @@ -32,29 +69,35 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets - node_security_group_additional_rules = { - # Control plane invoke Karpenter webhook - ingress_karpenter_webhook_tcp = { - description = "Control plane invoke Karpenter webhook" - protocol = "tcp" - from_port = 8443 - to_port = 8443 - type = "ingress" - source_cluster_security_group = true + # Fargate profiles use the cluster primary security group so these are not utilized + create_cluster_security_group = false + create_node_security_group = false + + manage_aws_auth_configmap = true + aws_auth_roles = [ + # We need to add in the Karpenter node IAM role for nodes launched by Karpenter + { + rolearn = module.karpenter.role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, + ] + + fargate_profiles = { + kube_system = { + name = "kube-system" + selectors = [ + { namespace = "kube-system" } + ] } - } - eks_managed_node_groups = { karpenter = { - instance_types = ["t3.medium"] - - min_size = 1 - max_size = 2 - desired_size = 1 - - iam_role_additional_policies = [ - # Required by Karpenter - "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + name = "karpenter" + selectors = [ + { namespace = "karpenter" } ] } } @@ -71,60 +114,14 @@ module "eks" { # Karpenter ################################################################################ -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] - } - } -} - -provider "kubectl" { - apply_retry_count = 5 - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - load_config_file = false - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] - } -} - -module "karpenter_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 4.21.1" - - role_name = "karpenter-controller-${local.name}" - attach_karpenter_controller_policy = true - karpenter_controller_cluster_id = module.eks.cluster_id - karpenter_controller_ssm_parameter_arns = [ - "arn:${local.partition}:ssm:*:*:parameter/aws/service/*" - ] - karpenter_controller_node_iam_role_arns = [ - module.eks.eks_managed_node_groups["karpenter"].iam_role_arn - ] +module "karpenter" { + source = "../../modules/karpenter" - oidc_providers = { - ex = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["karpenter:karpenter"] - } - } -} + cluster_name = module.eks.cluster_name + irsa_oidc_provider_arn = module.eks.oidc_provider_arn -resource "aws_iam_instance_profile" "karpenter" { - name = "KarpenterNodeInstanceProfile-${local.name}" - role = module.eks.eks_managed_node_groups["karpenter"].iam_role_name + tags = local.tags } resource "helm_release" "karpenter" { @@ -132,54 +129,73 @@ resource "helm_release" "karpenter" { create_namespace = true name = "karpenter" - repository = "https://charts.karpenter.sh" + repository = "oci://public.ecr.aws/karpenter" chart = "karpenter" - version = "0.8.2" + version = "v0.19.1" set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.karpenter_irsa.iam_role_arn + name = "settings.aws.clusterName" + value = module.eks.cluster_name } set { - name = "clusterName" - value = module.eks.cluster_id + name = "settings.aws.clusterEndpoint" + value = module.eks.cluster_endpoint } set { - name = "clusterEndpoint" - value = module.eks.cluster_endpoint + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.karpenter.irsa_arn } set { - name = "aws.defaultInstanceProfile" - value = aws_iam_instance_profile.karpenter.name + name = "settings.aws.defaultInstanceProfile" + value = module.karpenter.instance_profile_name + } + + set { + name = "settings.aws.interruptionQueueName" + value = module.karpenter.queue_name } } -# Workaround - https://github.com/hashicorp/terraform-provider-kubernetes/issues/1380#issuecomment-967022975 resource "kubectl_manifest" "karpenter_provisioner" { yaml_body = <<-YAML - apiVersion: karpenter.sh/v1alpha5 - kind: Provisioner - metadata: - name: default - spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] - limits: - resources: - cpu: 1000 - provider: + apiVersion: karpenter.sh/v1alpha5 + kind: Provisioner + metadata: + name: default + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + limits: + resources: + cpu: 1000 + providerRef: + name: default + ttlSecondsAfterEmpty: 30 + YAML + + depends_on = [ + helm_release.karpenter + ] +} + +resource "kubectl_manifest" "karpenter_node_template" { + yaml_body = <<-YAML + apiVersion: karpenter.k8s.aws/v1alpha1 + kind: AWSNodeTemplate + metadata: + name: default + spec: subnetSelector: - karpenter.sh/discovery: ${local.name} + karpenter.sh/discovery: ${module.eks.cluster_name} securityGroupSelector: - karpenter.sh/discovery: ${local.name} + karpenter.sh/discovery: ${module.eks.cluster_name} tags: - karpenter.sh/discovery: ${local.name} - ttlSecondsAfterEmpty: 30 + karpenter.sh/discovery: ${module.eks.cluster_name} YAML depends_on = [ @@ -191,27 +207,27 @@ resource "kubectl_manifest" "karpenter_provisioner" { # and starts with zero replicas resource "kubectl_manifest" "karpenter_example_deployment" { yaml_body = <<-YAML - apiVersion: apps/v1 - kind: Deployment - metadata: - name: inflate - spec: - replicas: 0 - selector: - matchLabels: - app: inflate - template: - metadata: - labels: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: inflate + spec: + replicas: 0 + selector: + matchLabels: app: inflate - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: inflate - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 - resources: - requests: - cpu: 1 + template: + metadata: + labels: + app: inflate + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 YAML depends_on = [ @@ -219,6 +235,129 @@ resource "kubectl_manifest" "karpenter_example_deployment" { ] } +################################################################################ +# Modify EKS CoreDNS Deployment +################################################################################ + +data "aws_eks_cluster_auth" "this" { + name = module.eks.cluster_name +} + +locals { + kubeconfig = yamlencode({ + apiVersion = "v1" + kind = "Config" + current-context = "terraform" + clusters = [{ + name = module.eks.cluster_name + cluster = { + certificate-authority-data = module.eks.cluster_certificate_authority_data + server = module.eks.cluster_endpoint + } + }] + contexts = [{ + name = "terraform" + context = { + cluster = module.eks.cluster_name + user = "terraform" + } + }] + users = [{ + name = "terraform" + user = { + token = data.aws_eks_cluster_auth.this.token + } + }] + }) +} + +# Separate resource so that this is only ever executed once +resource "null_resource" "remove_default_coredns_deployment" { + triggers = {} + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = base64encode(local.kubeconfig) + } + + # We are removing the deployment provided by the EKS service and replacing it through the self-managed CoreDNS Helm addon + # However, we are maintaing the existing kube-dns service and annotating it for Helm to assume control + command = <<-EOT + kubectl --namespace kube-system delete deployment coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) + EOT + } +} + +resource "null_resource" "modify_kube_dns" { + triggers = {} + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = base64encode(local.kubeconfig) + } + + # We are maintaing the existing kube-dns service and annotating it for Helm to assume control + command = <<-EOT + echo "Setting implicit dependency on ${module.eks.fargate_profiles["kube_system"].fargate_profile_pod_execution_role_arn}" + kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-name=coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) + kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-namespace=kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) + kubectl --namespace kube-system label --overwrite service kube-dns app.kubernetes.io/managed-by=Helm --kubeconfig <(echo $KUBECONFIG | base64 --decode) + EOT + } + + depends_on = [ + null_resource.remove_default_coredns_deployment + ] +} + +################################################################################ +# CoreDNS Helm Chart (self-managed) +################################################################################ + +data "aws_eks_addon_version" "this" { + for_each = toset(["coredns"]) + + addon_name = each.value + kubernetes_version = module.eks.cluster_version + most_recent = true +} + +resource "helm_release" "coredns" { + name = "coredns" + namespace = "kube-system" + create_namespace = false + description = "CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS Services" + chart = "coredns" + version = "1.19.4" + repository = "https://coredns.github.io/helm" + + # For EKS image repositories https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html + values = [ + <<-EOT + image: + repository: 602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/coredns + tag: ${data.aws_eks_addon_version.this["coredns"].version} + deployment: + name: coredns + annotations: + eks.amazonaws.com/compute-type: fargate + service: + name: kube-dns + annotations: + eks.amazonaws.com/compute-type: fargate + podAnnotations: + eks.amazonaws.com/compute-type: fargate + EOT + ] + + depends_on = [ + # Need to ensure the CoreDNS updates are peformed before provisioning + null_resource.modify_kube_dns + ] +} + ################################################################################ # Supporting Resources ################################################################################ diff --git a/examples/karpenter/outputs.tf b/examples/karpenter/outputs.tf index dbbec2334e..39d418bf37 100644 --- a/examples/karpenter/outputs.tf +++ b/examples/karpenter/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_name +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = module.eks.cluster_id } @@ -185,3 +190,93 @@ output "aws_auth_configmap_yaml" { description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" value = module.eks.aws_auth_configmap_yaml } + +################################################################################ +# IAM Role for Service Account (IRSA) +################################################################################ + +output "karpenter_irsa_name" { + description = "The name of the IAM role for service accounts" + value = module.karpenter.irsa_name +} + +output "karpenter_irsa_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role for service accounts" + value = module.karpenter.irsa_arn +} + +output "karpenter_irsa_unique_id" { + description = "Stable and unique string identifying the IAM role for service accounts" + value = module.karpenter.irsa_unique_id +} + +################################################################################ +# Node Termination Queue +################################################################################ + +output "karpenter_queue_arn" { + description = "The ARN of the SQS queue" + value = module.karpenter.queue_arn +} + +output "karpenter_queue_name" { + description = "The name of the created Amazon SQS queue" + value = module.karpenter.queue_name +} + +output "karpenter_queue_url" { + description = "The URL for the created Amazon SQS queue" + value = module.karpenter.queue_url +} + +################################################################################ +# Node Termination Event Rules +################################################################################ + +output "karpenter_event_rules" { + description = "Map of the event rules created and their attributes" + value = module.karpenter.event_rules +} + +################################################################################ +# Node IAM Role +################################################################################ + +output "karpenter_role_name" { + description = "The name of the IAM role" + value = module.karpenter.role_name +} + +output "karpenter_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = module.karpenter.role_arn +} + +output "karpenter_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.karpenter.role_unique_id +} + +################################################################################ +# Node IAM Instance Profile +################################################################################ + +output "karpenter_instance_profile_arn" { + description = "ARN assigned by AWS to the instance profile" + value = module.karpenter.instance_profile_arn +} + +output "karpenter_instance_profile_id" { + description = "Instance profile's ID" + value = module.karpenter.instance_profile_id +} + +output "karpenter_instance_profile_name" { + description = "Name of the instance profile" + value = module.karpenter.instance_profile_name +} + +output "karpenter_instance_profile_unique" { + description = "Stable and unique string identifying the IAM instance profile" + value = module.karpenter.instance_profile_unique +} diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index fe18abab81..92e85aacf6 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -6,6 +6,10 @@ terraform { source = "hashicorp/aws" version = ">= 3.72" } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } helm = { source = "hashicorp/helm" version = ">= 2.4" @@ -14,5 +18,9 @@ terraform { source = "gavinbunney/kubectl" version = ">= 1.14" } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } } } diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index a543d6454b..9608cdb8cd 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -77,8 +77,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 116dbc2553..27e82eebb3 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -10,13 +10,13 @@ provider "kubernetes" { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] } } locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.22" + cluster_version = "1.24" region = "eu-west-1" tags = { diff --git a/examples/self_managed_node_group/outputs.tf b/examples/self_managed_node_group/outputs.tf index 6e31908292..67de532779 100644 --- a/examples/self_managed_node_group/outputs.tf +++ b/examples/self_managed_node_group/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_name +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = module.eks.cluster_id } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index eb8f35dbd3..bf8d6de605 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -10,7 +10,7 @@ module "eks_managed_node_group" { name = "separate-eks-mng" cluster_name = "my-cluster" - cluster_version = "1.22" + cluster_version = "1.24" vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md new file mode 100644 index 0000000000..1797c6d2e8 --- /dev/null +++ b/modules/karpenter/README.md @@ -0,0 +1,195 @@ +# Karpenter Module + +Configuration in this directory creates the AWS resources required by Karpenter + +## Usage + +### All Resources (Default) + +In the following example, the Karpenter module will create: +- An IAM role for service accounts (IRSA) with a narrowly scoped IAM policy for the Karpenter controller to utilize +- An IAM role and instance profile for the nodes created by Karpenter to utilize + - Note: This IAM role ARN will need to be added to the `aws-auth` configmap for nodes to join the cluster successfully +- An SQS queue and Eventbridge event rules for Karpenter to utilize for spot termination handling, capacity rebalancing, etc. + +This setup is great for running Karpenter on EKS Fargate: + +```hcl +module "eks" { + source = "terraform-aws-modules/eks" + + # Shown just for connection between cluster and Karpenter sub-module below + manage_aws_auth_configmap = true + aws_auth_roles = [ + # We need to add in the Karpenter node IAM role for nodes launched by Karpenter + { + rolearn = module.karpenter.role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, + ] + ... +} + +module "karpenter" { + source = "terraform-aws-modules/eks/aws//modules/karpenter" + + cluster_name = module.eks.cluster_name + + irsa_oidc_provider_arn = module.eks.oidc_provider_arn + irsa_namespace_service_accounts = ["karpenter:karpenter"] + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### External Node IAM Role (Default) + +In the following example, the Karpenter module will create: +- An IAM role for service accounts (IRSA) with a narrowly scoped IAM policy for the Karpenter controller to utilize +- An IAM instance profile for the nodes created by Karpenter to utilize + - Note: This setup will utilize the existing IAM role created by the EKS Managed Node group which means the role is already populated in the `aws-auth` configmap and no further updates are required. +- An SQS queue and Eventbridge event rules for Karpenter to utilize for spot termination handling, capacity rebalancing, etc. + +In this scenario, Karpenter would run atop the EKS Managed Node group and scale out nodes as needed from there: + +```hcl +module "eks" { + source = "terraform-aws-modules/eks" + + # Shown just for connection between cluster and Karpenter sub-module below + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 1 + max_size = 3 + desired_size = 1 + } + } + ... +} + +module "karpenter" { + source = "terraform-aws-modules/eks/aws//modules/karpenter" + + cluster_name = module.eks.cluster_name + + irsa_oidc_provider_arn = module.eks.oidc_provider_arn + irsa_namespace_service_accounts = ["karpenter:karpenter"] + + create_iam_role = false + iam_role_arn = module.eks.eks_managed_node_groups["initial"].iam_role_arn + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.72 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.72 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_policy.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_sqs_queue.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | +| [aws_sqs_queue_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.irsa_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | +| [cluster\_name](#input\_cluster\_name) | The name of the EKS cluster | `string` | `""` | no | +| [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `true` | no | +| [create\_irsa](#input\_create\_irsa) | Determines whether an IAM role for service accounts is created | `bool` | `true` | no | +| [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_max\_session\_duration](#input\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `"/"` | no | +| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no | +| [irsa\_description](#input\_irsa\_description) | IAM role for service accounts description | `string` | `"Karpenter IAM role for service account"` | no | +| [irsa\_max\_session\_duration](#input\_irsa\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | +| [irsa\_name](#input\_irsa\_name) | Name of IAM role for service accounts | `string` | `null` | no | +| [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` |
[
"karpenter:karpenter"
]
| no | +| [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | +| [irsa\_path](#input\_irsa\_path) | Path of IAM role for service accounts | `string` | `"/"` | no | +| [irsa\_permissions\_boundary\_arn](#input\_irsa\_permissions\_boundary\_arn) | Permissions boundary ARN to use for IAM role for service accounts | `string` | `null` | no | +| [irsa\_ssm\_parameter\_arns](#input\_irsa\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter | `list(string)` |
[
"arn:aws:ssm:*:*:parameter/aws/service/*"
]
| no | +| [irsa\_subnet\_account\_id](#input\_irsa\_subnet\_account\_id) | Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account | `string` | `""` | no | +| [irsa\_tag\_key](#input\_irsa\_tag\_key) | Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner | `string` | `"karpenter.sh/discovery"` | no | +| [irsa\_tags](#input\_irsa\_tags) | A map of additional tags to add the the IAM role for service accounts | `map(any)` | `{}` | no | +| [irsa\_use\_name\_prefix](#input\_irsa\_use\_name\_prefix) | Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix | `bool` | `true` | no | +| [queue\_kms\_data\_key\_reuse\_period\_seconds](#input\_queue\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again | `number` | `null` | no | +| [queue\_kms\_master\_key\_id](#input\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no | +| [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | +| [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [event\_rules](#output\_event\_rules) | Map of the event rules created and their attributes | +| [instance\_profile\_arn](#output\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | +| [instance\_profile\_id](#output\_instance\_profile\_id) | Instance profile's ID | +| [instance\_profile\_name](#output\_instance\_profile\_name) | Name of the instance profile | +| [instance\_profile\_unique](#output\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | +| [irsa\_arn](#output\_irsa\_arn) | The Amazon Resource Name (ARN) specifying the IAM role for service accounts | +| [irsa\_name](#output\_irsa\_name) | The name of the IAM role for service accounts | +| [irsa\_unique\_id](#output\_irsa\_unique\_id) | Stable and unique string identifying the IAM role for service accounts | +| [queue\_arn](#output\_queue\_arn) | The ARN of the SQS queue | +| [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue | +| [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue | +| [role\_arn](#output\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [role\_name](#output\_role\_name) | The name of the IAM role | +| [role\_unique\_id](#output\_role\_unique\_id) | Stable and unique string identifying the IAM role | + diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf new file mode 100644 index 0000000000..c7ae5cc35d --- /dev/null +++ b/modules/karpenter/main.tf @@ -0,0 +1,359 @@ +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + +locals { + account_id = data.aws_caller_identity.current.account_id + partition = data.aws_partition.current.partition + dns_suffix = data.aws_partition.current.dns_suffix +} + +################################################################################ +# IAM Role for Service Account (IRSA) +# This is used by the Karpenter controller +################################################################################ + +locals { + create_irsa = var.create && var.create_irsa + irsa_name = coalesce(var.irsa_name, "KarpenterIRSA-${var.cluster_name}") + + irsa_oidc_provider_url = replace(var.irsa_oidc_provider_arn, "/^(.*provider/)/", "") +} + +data "aws_iam_policy_document" "irsa_assume_role" { + count = local.create_irsa ? 1 : 0 + + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = [var.irsa_oidc_provider_arn] + } + + condition { + test = var.irsa_assume_role_condition_test + variable = "${local.irsa_oidc_provider_url}:sub" + values = [for sa in var.irsa_namespace_service_accounts : "system:serviceaccount:${sa}"] + } + + # https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-oidc-and-irsa/?nc1=h_ls + condition { + test = var.irsa_assume_role_condition_test + variable = "${local.irsa_oidc_provider_url}:aud" + values = ["sts.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "irsa" { + count = local.create_irsa ? 1 : 0 + + name = var.irsa_use_name_prefix ? null : local.irsa_name + name_prefix = var.irsa_use_name_prefix ? "${local.irsa_name}-" : null + path = var.irsa_path + description = var.irsa_description + + assume_role_policy = data.aws_iam_policy_document.irsa_assume_role[0].json + max_session_duration = var.irsa_max_session_duration + permissions_boundary = var.irsa_permissions_boundary_arn + force_detach_policies = true + + tags = merge(var.tags, var.irsa_tags) +} + +data "aws_iam_policy_document" "irsa" { + count = local.create_irsa ? 1 : 0 + + statement { + actions = [ + "ec2:CreateLaunchTemplate", + "ec2:CreateFleet", + "ec2:CreateTags", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeSpotPriceHistory", + "pricing:GetProducts", + ] + + resources = ["*"] + } + + statement { + actions = [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate", + ] + + resources = ["*"] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/${var.irsa_tag_key}" + values = [var.cluster_name] + } + } + + statement { + actions = ["ec2:RunInstances"] + resources = [ + "arn:${local.partition}:ec2:*:${local.account_id}:launch-template/*", + ] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/${var.irsa_tag_key}" + values = [var.cluster_name] + } + } + + statement { + actions = ["ec2:RunInstances"] + resources = [ + "arn:${local.partition}:ec2:*::image/*", + "arn:${local.partition}:ec2:*:${local.account_id}:instance/*", + "arn:${local.partition}:ec2:*:${local.account_id}:spot-instances-request/*", + "arn:${local.partition}:ec2:*:${local.account_id}:security-group/*", + "arn:${local.partition}:ec2:*:${local.account_id}:volume/*", + "arn:${local.partition}:ec2:*:${local.account_id}:network-interface/*", + "arn:${local.partition}:ec2:*:${coalesce(var.irsa_subnet_account_id, local.account_id)}:subnet/*", + ] + } + + statement { + actions = ["ssm:GetParameter"] + resources = var.irsa_ssm_parameter_arns + } + + statement { + actions = ["iam:PassRole"] + resources = [var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn] + } + + dynamic "statement" { + for_each = local.enable_spot_termination ? [1] : [] + + content { + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:GetQueueAttributes", + "sqs:ReceiveMessage", + ] + resources = [aws_sqs_queue.this[0].arn] + } + } +} + +resource "aws_iam_policy" "irsa" { + count = var.create_irsa ? 1 : 0 + + name_prefix = "${local.irsa_name}-" + path = var.irsa_path + description = var.irsa_description + policy = data.aws_iam_policy_document.irsa[0].json + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "irsa" { + count = local.create_irsa ? 1 : 0 + + role = aws_iam_role.irsa[0].name + policy_arn = aws_iam_policy.irsa[0].arn +} + +################################################################################ +# Node Termination Queue +################################################################################ + +locals { + enable_spot_termination = var.create && var.enable_spot_termination + + queue_name = coalesce(var.queue_name, "Karpenter-${var.cluster_name}") +} + +resource "aws_sqs_queue" "this" { + count = local.enable_spot_termination ? 1 : 0 + + name = local.queue_name + message_retention_seconds = 300 + sqs_managed_sse_enabled = var.queue_managed_sse_enabled + kms_master_key_id = var.queue_kms_master_key_id + kms_data_key_reuse_period_seconds = var.queue_kms_data_key_reuse_period_seconds + + tags = var.tags +} + +data "aws_iam_policy_document" "queue" { + count = local.enable_spot_termination ? 1 : 0 + + statement { + sid = "SqsWrite" + actions = ["sqs:SendMessage"] + resources = [aws_sqs_queue.this[0].arn] + + principals { + type = "Service" + identifiers = [ + "events.${local.dns_suffix}", + "sqs.${local.dns_suffix}", + ] + } + + } +} + +resource "aws_sqs_queue_policy" "this" { + count = local.enable_spot_termination ? 1 : 0 + + queue_url = aws_sqs_queue.this[0].url + policy = data.aws_iam_policy_document.queue[0].json +} + +################################################################################ +# Node Termination Event Rules +################################################################################ + +locals { + events = { + health_event = { + name = "HealthEvent" + description = "Karpenter interrupt - AWS health event" + event_pattern = { + source = ["aws.health"] + detail-type = ["AWS Health Event"] + } + } + spot_interupt = { + name = "SpotInterrupt" + description = "Karpenter interrupt - EC2 spot instance interruption warning" + event_pattern = { + source = ["aws.ec2"] + detail-type = ["EC2 Spot Instance Interruption Warning"] + } + } + instance_rebalance = { + name = "InstanceRebalance" + description = "Karpenter interrupt - EC2 instance rebalance recommendation" + event_pattern = { + source = ["aws.ec2"] + detail-type = ["EC2 Instance Rebalance Recommendation"] + } + } + instance_state_change = { + name = "InstanceStateChange" + description = "Karpenter interrupt - EC2 instance state-change notification" + event_pattern = { + source = ["aws.ec2"] + detail-type = ["EC2 Instance State-change Notification"] + } + } + } +} + +resource "aws_cloudwatch_event_rule" "this" { + for_each = { for k, v in local.events : k => v if local.enable_spot_termination } + + name = "Karpenter${each.value.name}-${var.cluster_name}" + description = each.value.description + event_pattern = jsonencode(each.value.event_pattern) + + tags = var.tags +} + +resource "aws_cloudwatch_event_target" "this" { + for_each = { for k, v in local.events : k => v if local.enable_spot_termination } + + rule = aws_cloudwatch_event_rule.this[each.key].name + target_id = "KarpenterInterruptionQueueTarget" + arn = aws_sqs_queue.this[0].arn +} + +################################################################################ +# Node IAM Role +# This is used by the nodes launched by Karpenter +################################################################################ + +locals { + create_iam_role = var.create && var.create_iam_role + + iam_role_name = coalesce(var.iam_role_name, "Karpenter-${var.cluster_name}") + iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" + cni_policy = var.cluster_ip_family == "ipv6" ? "${local.iam_role_policy_prefix}/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" +} + +data "aws_iam_policy_document" "assume_role" { + count = local.create_iam_role ? 1 : 0 + + statement { + sid = "EKSNodeAssumeRole" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.${local.dns_suffix}"] + } + } +} + +resource "aws_iam_role" "this" { + count = local.create_iam_role ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + assume_role_policy = data.aws_iam_policy_document.assume_role[0].json + max_session_duration = var.iam_role_max_session_duration + permissions_boundary = var.iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +# Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group +resource "aws_iam_role_policy_attachment" "this" { + for_each = { for k, v in toset(compact([ + "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", + "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", + var.iam_role_attach_cni_policy ? local.cni_policy : "", + ])) : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +################################################################################ +# Node IAM Instance Profile +# This is used by the nodes launched by Karpenter +################################################################################ + +locals { + external_role_name = try(replace(var.iam_role_arn, "/^(.*role/)/", ""), null) +} + +resource "aws_iam_instance_profile" "this" { + count = var.create && var.create_instance_profile ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + role = var.create_iam_role ? aws_iam_role.this[0].name : local.external_role_name + + tags = merge(var.tags, var.iam_role_tags) +} diff --git a/modules/karpenter/outputs.tf b/modules/karpenter/outputs.tf new file mode 100644 index 0000000000..947de39bfd --- /dev/null +++ b/modules/karpenter/outputs.tf @@ -0,0 +1,89 @@ +################################################################################ +# IAM Role for Service Account (IRSA) +################################################################################ + +output "irsa_name" { + description = "The name of the IAM role for service accounts" + value = try(aws_iam_role.irsa[0].name, null) +} + +output "irsa_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role for service accounts" + value = try(aws_iam_role.irsa[0].arn, null) +} + +output "irsa_unique_id" { + description = "Stable and unique string identifying the IAM role for service accounts" + value = try(aws_iam_role.irsa[0].unique_id, null) +} + +################################################################################ +# Node Termination Queue +################################################################################ + +output "queue_arn" { + description = "The ARN of the SQS queue" + value = try(aws_sqs_queue.this[0].arn, null) +} + +output "queue_name" { + description = "The name of the created Amazon SQS queue" + value = try(aws_sqs_queue.this[0].name, null) +} + +output "queue_url" { + description = "The URL for the created Amazon SQS queue" + value = try(aws_sqs_queue.this[0].url, null) +} + +################################################################################ +# Node Termination Event Rules +################################################################################ + +output "event_rules" { + description = "Map of the event rules created and their attributes" + value = aws_cloudwatch_event_rule.this +} + +################################################################################ +# Node IAM Role +################################################################################ + +output "role_name" { + description = "The name of the IAM role" + value = try(aws_iam_role.this[0].name, null) +} + +output "role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = try(aws_iam_role.this[0].arn, var.iam_role_arn) +} + +output "role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, null) +} + +################################################################################ +# Node IAM Instance Profile +################################################################################ + +output "instance_profile_arn" { + description = "ARN assigned by AWS to the instance profile" + value = try(aws_iam_instance_profile.this[0].arn, null) +} + +output "instance_profile_id" { + description = "Instance profile's ID" + value = try(aws_iam_instance_profile.this[0].id, null) +} + +output "instance_profile_name" { + description = "Name of the instance profile" + value = try(aws_iam_instance_profile.this[0].name, null) +} + +output "instance_profile_unique" { + description = "Stable and unique string identifying the IAM instance profile" + value = try(aws_iam_instance_profile.this[0].unique_id, null) +} diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf new file mode 100644 index 0000000000..a6327ba932 --- /dev/null +++ b/modules/karpenter/variables.tf @@ -0,0 +1,226 @@ +variable "create" { + description = "Determines whether to create EKS managed node group or not" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +variable "cluster_name" { + description = "The name of the EKS cluster" + type = string + default = "" +} + +################################################################################ +# IAM Role for Service Account (IRSA) +################################################################################ + +variable "create_irsa" { + description = "Determines whether an IAM role for service accounts is created" + type = bool + default = true +} + +variable "irsa_name" { + description = "Name of IAM role for service accounts" + type = string + default = null +} + +variable "irsa_use_name_prefix" { + description = "Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix" + type = bool + default = true +} + +variable "irsa_path" { + description = "Path of IAM role for service accounts" + type = string + default = "/" +} + +variable "irsa_description" { + description = "IAM role for service accounts description" + type = string + default = "Karpenter IAM role for service account" +} + +variable "irsa_max_session_duration" { + description = "Maximum API session duration in seconds between 3600 and 43200" + type = number + default = null +} + +variable "irsa_permissions_boundary_arn" { + description = "Permissions boundary ARN to use for IAM role for service accounts" + type = string + default = null +} + +variable "irsa_tags" { + description = "A map of additional tags to add the the IAM role for service accounts" + type = map(any) + default = {} +} + +variable "irsa_tag_key" { + description = "Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner" + type = string + default = "karpenter.sh/discovery" +} + +variable "irsa_ssm_parameter_arns" { + description = "List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter" + type = list(string) + # https://github.com/aws/karpenter/blob/ed9473a9863ca949b61b9846c8b9f33f35b86dbd/pkg/cloudprovider/aws/ami.go#L105-L123 + default = ["arn:aws:ssm:*:*:parameter/aws/service/*"] +} + +variable "irsa_subnet_account_id" { + description = "Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account" + type = string + default = "" +} + +variable "irsa_oidc_provider_arn" { + description = "OIDC provider arn used in trust policy for IAM role for service accounts" + type = string + default = "" +} + +variable "irsa_namespace_service_accounts" { + description = "List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts" + type = list(string) + default = ["karpenter:karpenter"] +} + +variable "irsa_assume_role_condition_test" { + description = "Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role" + type = string + default = "StringEquals" +} + +################################################################################ +# Node Termination Queue +################################################################################ + +variable "enable_spot_termination" { + description = "Determines whether to enable native spot termination handling" + type = bool + default = true +} + +variable "queue_name" { + description = "Name of the SQS queue" + type = string + default = null +} + +variable "queue_managed_sse_enabled" { + description = "Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys" + type = bool + default = true +} + +variable "queue_kms_master_key_id" { + description = "The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK" + type = string + default = null +} + +variable "queue_kms_data_key_reuse_period_seconds" { + description = "The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again" + type = number + default = null +} + +################################################################################ +# Node IAM Role & Instance Profile +################################################################################ + +variable "create_iam_role" { + description = "Determines whether an IAM role is created or to use an existing IAM role" + type = bool + default = true +} + +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = null +} + +variable "iam_role_arn" { + description = "Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false`" + type = string + default = null +} + +variable "iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "iam_role_path" { + description = "IAM role path" + type = string + default = "/" +} + +variable "iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "iam_role_max_session_duration" { + description = "Maximum API session duration in seconds between 3600 and 43200" + type = number + default = null +} + +variable "iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "iam_role_attach_cni_policy" { + description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" + type = bool + default = true +} + +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = list(string) + default = [] +} + +variable "iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} + +################################################################################ +# Node IAM Instance Profile +################################################################################ + +variable "create_instance_profile" { + description = "Whether to create an IAM instance profile" + type = bool + default = true +} diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf new file mode 100644 index 0000000000..22e8d7265f --- /dev/null +++ b/modules/karpenter/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.72" + } + } +} diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index eb0b484443..4c392362f6 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -10,7 +10,7 @@ module "self_managed_node_group" { name = "separate-self-mng" cluster_name = "my-cluster" - cluster_version = "1.22" + cluster_version = "1.24" cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" diff --git a/outputs.tf b/outputs.tf index 7fb6d8ce98..eadf2d0cd2 100644 --- a/outputs.tf +++ b/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = try(aws_eks_cluster.this[0].endpoint, "") } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = try(aws_eks_cluster.this[0].name, "") +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = try(aws_eks_cluster.this[0].id, "") } diff --git a/variables.tf b/variables.tf index df2ee51638..6bce0560e2 100644 --- a/variables.tf +++ b/variables.tf @@ -27,7 +27,7 @@ variable "cluster_name" { } variable "cluster_version" { - description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.22`)" + description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.24`)" type = string default = null } From fe85170cab236c98af70197fd60bf8332cf0509f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 21 Nov 2022 18:51:07 +0000 Subject: [PATCH 603/962] chore(release): version 18.31.0 [skip ci] ## [18.31.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.3...v18.31.0) (2022-11-21) ### Features * New Karpenter sub-module for easily enabling Karpenter on EKS ([#2303](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2303)) ([f24de33](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f24de3326d3c12ce61fbaefe1e3dbe7418d8bc85)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 774a3bc7f1..d96c76b830 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [18.31.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.3...v18.31.0) (2022-11-21) + + +### Features + +* New Karpenter sub-module for easily enabling Karpenter on EKS ([#2303](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2303)) ([f24de33](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f24de3326d3c12ce61fbaefe1e3dbe7418d8bc85)) + ### [18.30.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.2...v18.30.3) (2022-11-07) From 7436178cc1a720a066c73f1de23b04b3c24ae608 Mon Sep 17 00:00:00 2001 From: ThetaSinner Date: Tue, 22 Nov 2022 16:16:39 +0000 Subject: [PATCH 604/962] fix: Include all certificate fingerprints in the OIDC provider thumbprint list (#2307) Co-authored-by: gdjensen --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index a17d4a1499..3ed4edaacf 100644 --- a/main.tf +++ b/main.tf @@ -213,7 +213,7 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { count = local.create && var.enable_irsa ? 1 : 0 client_id_list = distinct(compact(concat(["sts.${local.dns_suffix}"], var.openid_connect_audiences))) - thumbprint_list = concat([data.tls_certificate.this[0].certificates[0].sha1_fingerprint], var.custom_oidc_thumbprints) + thumbprint_list = concat(data.tls_certificate.this[0].certificates[*].sha1_fingerprint, var.custom_oidc_thumbprints) url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer tags = merge( From 000dd2c2ee68701c5f6be9c9fa739120c1995348 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 22 Nov 2022 16:17:20 +0000 Subject: [PATCH 605/962] chore(release): version 18.31.1 [skip ci] ### [18.31.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.31.0...v18.31.1) (2022-11-22) ### Bug Fixes * Include all certificate fingerprints in the OIDC provider thumbprint list ([#2307](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2307)) ([7436178](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7436178cc1a720a066c73f1de23b04b3c24ae608)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d96c76b830..e1532f6e2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.31.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.31.0...v18.31.1) (2022-11-22) + + +### Bug Fixes + +* Include all certificate fingerprints in the OIDC provider thumbprint list ([#2307](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2307)) ([7436178](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7436178cc1a720a066c73f1de23b04b3c24ae608)) + ## [18.31.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.30.3...v18.31.0) (2022-11-21) From 3fb28b357f4fc9144340f94abe9dd520e89f49e2 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 23 Nov 2022 12:43:42 -0500 Subject: [PATCH 606/962] fix: Ensure that `var.create` is tied to all resources correctly (#2308) --- modules/karpenter/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index c7ae5cc35d..d54fa9585a 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -152,7 +152,7 @@ data "aws_iam_policy_document" "irsa" { } resource "aws_iam_policy" "irsa" { - count = var.create_irsa ? 1 : 0 + count = local.create_irsa ? 1 : 0 name_prefix = "${local.irsa_name}-" path = var.irsa_path From efbe9526322cef7443734c6c8b1e2c166f1cd428 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 23 Nov 2022 17:44:27 +0000 Subject: [PATCH 607/962] chore(release): version 18.31.2 [skip ci] ### [18.31.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.31.1...v18.31.2) (2022-11-23) ### Bug Fixes * Ensure that `var.create` is tied to all resources correctly ([#2308](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2308)) ([3fb28b3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3fb28b357f4fc9144340f94abe9dd520e89f49e2)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1532f6e2a..c48e32e975 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [18.31.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.31.1...v18.31.2) (2022-11-23) + + +### Bug Fixes + +* Ensure that `var.create` is tied to all resources correctly ([#2308](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2308)) ([3fb28b3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3fb28b357f4fc9144340f94abe9dd520e89f49e2)) + ### [18.31.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.31.0...v18.31.1) (2022-11-22) From b2e97ca3dcbcd76063f1c932aa5199b4f49a2aa1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 5 Dec 2022 16:26:23 -0500 Subject: [PATCH 608/962] feat!: Add support for Outposts, remove node security group, add support for addon `preserve` and `most_recent` configurations (#2250) Co-authored-by: Anton Babenko Resolves undefined --- .github/images/security_groups.svg | 2 +- .pre-commit-config.yaml | 4 +- README.md | 93 +-- docs/UPGRADE-19.0.md | 466 +++++++++++++ docs/compute_resources.md | 48 +- docs/faq.md | 36 - docs/network_connectivity.md | 3 +- docs/user_data.md | 18 +- examples/complete/README.md | 12 +- examples/complete/main.tf | 115 ++-- examples/complete/outputs.tf | 12 +- examples/complete/versions.tf | 4 +- examples/eks_managed_node_group/README.md | 26 +- examples/eks_managed_node_group/main.tf | 456 +++---------- examples/eks_managed_node_group/outputs.tf | 12 +- examples/eks_managed_node_group/versions.tf | 8 +- examples/fargate_profile/README.md | 13 +- examples/fargate_profile/main.tf | 64 +- examples/fargate_profile/outputs.tf | 12 +- examples/fargate_profile/versions.tf | 4 +- examples/karpenter/README.md | 17 +- examples/karpenter/main.tf | 59 +- examples/karpenter/outputs.tf | 12 +- examples/karpenter/versions.tf | 6 +- examples/outposts/README.md | 115 ++++ examples/outposts/main.tf | 152 +++++ examples/outposts/outputs.tf | 211 ++++++ examples/outposts/prerequisites/main.tf | 150 +++++ examples/outposts/prerequisites/outputs.tf | 4 + examples/outposts/prerequisites/variables.tf | 5 + examples/outposts/prerequisites/versions.tf | 10 + examples/outposts/variables.tf | 5 + examples/outposts/versions.tf | 14 + examples/self_managed_node_group/README.md | 22 +- examples/self_managed_node_group/main.tf | 313 +++------ examples/self_managed_node_group/outputs.tf | 12 +- examples/self_managed_node_group/versions.tf | 8 +- examples/user_data/README.md | 3 +- examples/user_data/main.tf | 64 +- examples/user_data/versions.tf | 9 +- main.tf | 152 +++-- modules/_user_data/README.md | 2 +- modules/_user_data/outputs.tf | 2 +- modules/_user_data/versions.tf | 2 +- modules/eks-managed-node-group/README.md | 50 +- modules/eks-managed-node-group/main.tf | 303 +++++---- modules/eks-managed-node-group/outputs.tf | 34 +- modules/eks-managed-node-group/variables.tf | 98 +-- modules/eks-managed-node-group/versions.tf | 4 +- modules/fargate-profile/README.md | 13 +- modules/fargate-profile/main.tf | 18 +- modules/fargate-profile/outputs.tf | 12 +- modules/fargate-profile/variables.tf | 4 +- modules/fargate-profile/versions.tf | 4 +- modules/karpenter/README.md | 6 +- modules/karpenter/main.tf | 7 +- modules/karpenter/versions.tf | 4 +- modules/self-managed-node-group/README.md | 43 +- modules/self-managed-node-group/main.tf | 649 ++++++++++++------- modules/self-managed-node-group/outputs.tf | 58 +- modules/self-managed-node-group/variables.tf | 119 ++-- modules/self-managed-node-group/versions.tf | 4 +- node_groups.tf | 224 +++---- outputs.tf | 52 +- variables.tf | 60 +- versions.tf | 4 +- 66 files changed, 2750 insertions(+), 1777 deletions(-) create mode 100644 docs/UPGRADE-19.0.md create mode 100644 examples/outposts/README.md create mode 100644 examples/outposts/main.tf create mode 100644 examples/outposts/outputs.tf create mode 100644 examples/outposts/prerequisites/main.tf create mode 100644 examples/outposts/prerequisites/outputs.tf create mode 100644 examples/outposts/prerequisites/variables.tf create mode 100644 examples/outposts/prerequisites/versions.tf create mode 100644 examples/outposts/variables.tf create mode 100644 examples/outposts/versions.tf diff --git a/.github/images/security_groups.svg b/.github/images/security_groups.svg index 6b120e98ba..6012962597 100644 --- a/.github/images/security_groups.svg +++ b/.github/images/security_groups.svg @@ -1 +1 @@ - + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74f3751c74..d5886a6d32 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.76.0 + rev: v1.77.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -23,7 +23,7 @@ repos: - '--args=--only=terraform_standard_module_structure' - '--args=--only=terraform_workspace_remote' - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer diff --git a/README.md b/README.md index d003e14724..fb7dd4845f 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Terraform module which creates AWS EKS (Kubernetes) resources - Upgrade Guides - [Upgrade to v17.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-17.0.md) - [Upgrade to v18.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-18.0.md) + - [Upgrade to v19.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-19.0.md) ### External Documentation @@ -21,10 +22,17 @@ Please note that we strive to provide a comprehensive suite of documentation for - [AWS EKS Documentation](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Kubernetes Documentation](https://kubernetes.io/docs/home/) +#### Reference Architecture + +The examples provided under `examples/` provide a comprehensive suite of configurations that demonstrate nearly all of the possible different configurations and settings that can be used with this module. However, these examples are not representative of clusters that you would normally find in use for production workloads. For reference architectures that utilize this module, please see the following: + +- [EKS Reference Architecture](https://github.com/clowdhaus/eks-reference-architecture) + ## Available Features - AWS EKS Cluster Addons - AWS EKS Identity Provider Configuration +- [AWS EKS on Outposts support](https://aws.amazon.com/blogs/aws/deploy-your-amazon-eks-clusters-locally-on-aws-outposts/) - All [node types](https://docs.aws.amazon.com/eks/latest/userguide/eks-compute.html) are supported: - [EKS Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) - [Self Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) @@ -43,13 +51,22 @@ An IAM role for service accounts (IRSA) sub-module has been created to make depl Some of the addon/controller policies that are currently supported include: +- [Cert-Manager](https://cert-manager.io/docs/configuration/acme/dns01/route53/#set-up-an-iam-role) - [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md) -- [External DNS](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#iam-policy) - [EBS CSI Driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/example-iam-policy.json) -- [VPC CNI](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html) -- [Node Termination Handler](https://github.com/aws/aws-node-termination-handler#5-create-an-iam-role-for-the-pods) -- [Karpenter](https://karpenter.sh/preview/getting-started/getting-started-with-terraform/#create-the-karpentercontroller-iam-role) +- [EFS CSI Driver](https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/iam-policy-example.json) +- [External DNS](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#iam-policy) +- [External Secrets](https://github.com/external-secrets/kubernetes-external-secrets#add-a-secret) +- [FSx for Lustre CSI Driver](https://github.com/kubernetes-sigs/aws-fsx-csi-driver/blob/master/docs/README.md) +- [Karpenter](https://github.com/aws/karpenter/blob/main/website/content/en/preview/getting-started/cloudformation.yaml) - [Load Balancer Controller](https://github.com/kubernetes-sigs/aws-load-balancer-controller/blob/main/docs/install/iam_policy.json) + - [Load Balancer Controller Target Group Binding Only](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/deploy/installation/#iam-permission-subset-for-those-who-use-targetgroupbinding-only-and-dont-plan-to-use-the-aws-load-balancer-controller-to-manage-security-group-rules) +- [App Mesh Controller](https://github.com/aws/aws-app-mesh-controller-for-k8s/blob/master/config/iam/controller-iam-policy.json) + - [App Mesh Envoy Proxy](https://raw.githubusercontent.com/aws/aws-app-mesh-controller-for-k8s/master/config/iam/envoy-iam-policy.json) +- [Managed Service for Prometheus](https://docs.aws.amazon.com/prometheus/latest/userguide/set-up-irsa.html) +- [Node Termination Handler](https://github.com/aws/aws-node-termination-handler#5-create-an-iam-role-for-the-pods) +- [Velero](https://github.com/vmware-tanzu/velero-plugin-for-aws#option-1-set-permissions-with-an-iam-user) +- [VPC CNI](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html) See [terraform-aws-iam/modules/iam-role-for-service-accounts](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) for current list of supported addon/controller policies as more are added to the project. @@ -58,39 +75,36 @@ See [terraform-aws-iam/modules/iam-role-for-service-accounts](https://github.com ```hcl module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 18.0" + version = "~> 19.0" cluster_name = "my-cluster" cluster_version = "1.24" - cluster_endpoint_private_access = true cluster_endpoint_public_access = true cluster_addons = { coredns = { - resolve_conflicts = "OVERWRITE" + most_recent = true + } + kube-proxy = { + most_recent = true } - kube-proxy = {} vpc-cni = { - resolve_conflicts = "OVERWRITE" + most_recent = true } } - cluster_encryption_config = [{ - provider_key_arn = "arn:aws:kms:eu-west-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab" - resources = ["secrets"] - }] - - vpc_id = "vpc-1234556abcdef" - subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + control_plane_subnet_ids = ["subnet-xyzde987", "subnet-slkjf456", "subnet-qeiru789"] # Self Managed Node Group(s) self_managed_node_group_defaults = { instance_type = "m6i.large" update_launch_template_default_version = true - iam_role_additional_policies = [ - "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" - ] + iam_role_additional_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } } self_managed_node_groups = { @@ -123,7 +137,6 @@ module "eks" { # EKS Managed Node Group(s) eks_managed_node_group_defaults = { - disk_size = 50 instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] } @@ -192,7 +205,8 @@ module "eks" { - [Complete](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete): EKS Cluster using all available node group types in various combinations demonstrating many of the supported features and configurations - [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group): EKS Cluster using EKS managed node groups - [Fargate Profile](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile): EKS cluster using [Fargate Profiles](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) -- [Karpenter](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/karpenter): EKS Cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling +- [Karpenter](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/karpenter): EKS Cluster with [Karpenter](https://karpenter.sh/) provisioned for intelligent data plane management +- [Outposts](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/outposts): EKS local cluster provisioned on [AWS Outposts](https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts.html) - [Self Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self_managed_node_group): EKS Cluster using self-managed node groups - [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data): Various supported methods of providing necessary bootstrap scripts and configuration settings via user data @@ -208,8 +222,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | | [tls](#requirement\_tls) | >= 3.0 | @@ -217,7 +231,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.72 | +| [aws](#provider\_aws) | >= 4.45 | | [kubernetes](#provider\_kubernetes) | >= 2.10 | | [tls](#provider\_tls) | >= 3.0 | @@ -227,7 +241,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|--------|---------| | [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ./modules/eks-managed-node-group | n/a | | [fargate\_profile](#module\_fargate\_profile) | ./modules/fargate-profile | n/a | -| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 1.0.2 | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 1.1.0 | | [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ./modules/self-managed-node-group | n/a | ## Resources @@ -243,6 +257,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_iam_policy.cluster_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cluster_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_security_group.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | @@ -252,7 +267,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | | [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_default_tags.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | +| [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -273,15 +288,16 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days | `number` | `90` | no | | [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | | [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no | +| [cluster\_addons\_timeouts](#input\_cluster\_addons\_timeouts) | Create, update, and delete timeout configurations for the cluster addons | `map(string)` | `{}` | no | | [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | -| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster | `list(any)` | `[]` | no | +| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster | `any` |
{
"resources": [
"secrets"
]
}
| no | | [cluster\_encryption\_policy\_description](#input\_cluster\_encryption\_policy\_description) | Description of the cluster encryption policy created | `string` | `"Cluster encryption policy to allow cluster role to utilize CMK provided"` | no | | [cluster\_encryption\_policy\_name](#input\_cluster\_encryption\_policy\_name) | Name to use on cluster encryption policy created | `string` | `null` | no | | [cluster\_encryption\_policy\_path](#input\_cluster\_encryption\_policy\_path) | Cluster encryption policy path | `string` | `null` | no | | [cluster\_encryption\_policy\_tags](#input\_cluster\_encryption\_policy\_tags) | A map of additional tags to add to the cluster encryption policy created | `map(string)` | `{}` | no | | [cluster\_encryption\_policy\_use\_name\_prefix](#input\_cluster\_encryption\_policy\_use\_name\_prefix) | Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix | `bool` | `true` | no | -| [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `false` | no | -| [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `true` | no | +| [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `true` | no | +| [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `false` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [cluster\_iam\_role\_dns\_suffix](#input\_cluster\_iam\_role\_dns\_suffix) | Base DNS domain name for the current partition (e.g., amazonaws.com in AWS Commercial, amazonaws.com.cn in AWS China) | `string` | `null` | no | | [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | @@ -289,11 +305,12 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | | [cluster\_security\_group\_additional\_rules](#input\_cluster\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created. Set `source_node_security_group = true` inside rules to set the `node_security_group` as source | `any` | `{}` | no | | [cluster\_security\_group\_description](#input\_cluster\_security\_group\_description) | Description of the cluster security group created | `string` | `"EKS cluster security group"` | no | -| [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Existing security group ID to be attached to the cluster. Required if `create_cluster_security_group` = `false` | `string` | `""` | no | +| [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Existing security group ID to be attached to the cluster | `string` | `""` | no | | [cluster\_security\_group\_name](#input\_cluster\_security\_group\_name) | Name to use on cluster security group created | `string` | `null` | no | | [cluster\_security\_group\_tags](#input\_cluster\_security\_group\_tags) | A map of additional tags to add to the cluster security group created | `map(string)` | `{}` | no | | [cluster\_security\_group\_use\_name\_prefix](#input\_cluster\_security\_group\_use\_name\_prefix) | Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix | `bool` | `true` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | +| [cluster\_service\_ipv6\_cidr](#input\_cluster\_service\_ipv6\_cidr) | The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.24`) | `string` | `null` | no | @@ -302,10 +319,10 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | | [create\_cluster\_primary\_security\_group\_tags](#input\_create\_cluster\_primary\_security\_group\_tags) | Indicates whether or not to tag the cluster's primary security group. This security group is created by the EKS service, not the module, and therefore tagging is handled after cluster creation | `bool` | `true` | no | -| [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster or use the existing `cluster_security_group_id` | `bool` | `true` | no | +| [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default | `bool` | `true` | no | | [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy) | `bool` | `false` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether a an IAM role is created or to use an existing IAM role | `bool` | `true` | no | -| [create\_kms\_key](#input\_create\_kms\_key) | Controls if a KMS key for cluster encryption should be created | `bool` | `false` | no | +| [create\_kms\_key](#input\_create\_kms\_key) | Controls if a KMS key for cluster encryption should be created | `bool` | `true` | no | | [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | | [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | @@ -314,7 +331,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled. Defaults to `true` | `bool` | `true` | no | | [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no | | [fargate\_profiles](#input\_fargate\_profiles) | Map of Fargate Profile definitions to create | `any` | `{}` | no | -| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the cluster. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | @@ -335,20 +352,20 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [manage\_aws\_auth\_configmap](#input\_manage\_aws\_auth\_configmap) | Determines whether to manage the aws-auth configmap | `bool` | `false` | no | | [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no | | [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | +| [node\_security\_group\_enable\_recommended\_rules](#input\_node\_security\_group\_enable\_recommended\_rules) | Determines whether to enable recommended security group rules for the node security group created. This includes node-to-node TCP ingress on ephemeral ports and allows all egress traffic | `bool` | `true` | no | | [node\_security\_group\_id](#input\_node\_security\_group\_id) | ID of an existing security group to attach to the node groups created | `string` | `""` | no | | [node\_security\_group\_name](#input\_node\_security\_group\_name) | Name to use on node security group created | `string` | `null` | no | -| [node\_security\_group\_ntp\_ipv4\_cidr\_block](#input\_node\_security\_group\_ntp\_ipv4\_cidr\_block) | IPv4 CIDR block to allow NTP egress. Default is public IP space, but [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) can be used as well with `["169.254.169.123/32"]` | `list(string)` |
[
"0.0.0.0/0"
]
| no | -| [node\_security\_group\_ntp\_ipv6\_cidr\_block](#input\_node\_security\_group\_ntp\_ipv6\_cidr\_block) | IPv4 CIDR block to allow NTP egress. Default is public IP space, but [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) can be used as well with `["fd00:ec2::123/128"]` | `list(string)` |
[
"::/0"
]
| no | | [node\_security\_group\_tags](#input\_node\_security\_group\_tags) | A map of additional tags to add to the node security group created | `map(string)` | `{}` | no | | [node\_security\_group\_use\_name\_prefix](#input\_node\_security\_group\_use\_name\_prefix) | Determines whether node security group name (`node_security_group_name`) is used as a prefix | `bool` | `true` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider | `list(string)` | `[]` | no | +| [outpost\_config](#input\_outpost\_config) | Configuration for the AWS Outpost to provision the cluster on | `any` | `{}` | no | | [prefix\_separator](#input\_prefix\_separator) | The separator to use between the prefix and the generated timestamp for resource names | `string` | `"-"` | no | | [putin\_khuylo](#input\_putin\_khuylo) | Do you agree that Putin doesn't respect Ukrainian sovereignty and territorial integrity? More info: https://en.wikipedia.org/wiki/Putin_khuylo! | `bool` | `true` | no | | [self\_managed\_node\_group\_defaults](#input\_self\_managed\_node\_group\_defaults) | Map of self-managed node group default configurations | `any` | `{}` | no | | [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create | `any` | `{}` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets | `list(string)` | `[]` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | -| [vpc\_id](#input\_vpc\_id) | ID of the VPC where the cluster and its nodes will be provisioned | `string` | `null` | no | +| [vpc\_id](#input\_vpc\_id) | ID of the VPC where the cluster security group will be provisioned | `string` | `null` | no | ## Outputs @@ -364,9 +381,9 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | -| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/docs/UPGRADE-19.0.md b/docs/UPGRADE-19.0.md new file mode 100644 index 0000000000..429a6a8cbd --- /dev/null +++ b/docs/UPGRADE-19.0.md @@ -0,0 +1,466 @@ +# Upgrade from v18.x to v19.x + +Please consult the `examples` directory for reference example configurations. If you find a bug, please open an issue with supporting configuration to reproduce. + +## List of backwards incompatible changes + +- The `cluster_id` output used to output the name of the cluster. This is due to the fact that the cluster name is a unique constraint and therefore its set as the unique identifier within Terraform's state map. However, starting with local EKS clusters created on Outposts, there is now an attribute returned from the `aws eks create-cluster` API named `id`. The `cluster_id` has been updated to return this value which means that for current, standard EKS clusters created in the AWS cloud, no value will be returned (at the time of this writing) for `cluster_id` and only local EKS clusters on Outposts will return a value that looks like a UUID/GUID. Users should switch all instances of `cluster_id` to use `cluster_name` before upgrading to v19. [Reference](https://github.com/hashicorp/terraform-provider-aws/issues/27560) +- Minimum supported version of Terraform AWS provider updated to v4.45 to support latest features provided via the resources utilized. +- Minimum supported version of Terraform updated to v1.0 +- Individual security group created per EKS managed node group or self managed node group has been removed. This configuration went mostly un-used and would often cause confusion ("Why is there an empty security group attached to my nodes?"). This functionality can easily be replicated by user's providing one or more externally created security groups to attach to nodes launched from the node group. +- Previously, `var.iam_role_additional_policies` (one for each of the following: cluster IAM role, EKS managed node group IAM role, self-managed node group IAM role, and Fargate Profile IAM role) accepted a list of strings. This worked well for policies that already existed but failed for policies being created at the same time as the cluster due to the well known issue of unkown values used in a `for_each` loop. To rectify this issue in `v19.x`, two changes were made: + 1. `var.iam_role_additional_policies` was changed from type `list(string)` to type `map(string)` -> this is a breaking change. More information on managing this change can be found below, under `Terraform State Moves` + 2. The logic used in the root module for this variable was changed to replace the use of `try()` with `lookup()`. More details on why can be found [here](https://github.com/clowdhaus/terraform-for-each-unknown) +- The cluster name has been removed from the Karpenter module event rule names. Due to the use of long cluster names appending to the provided naming scheme, the cluster name has moved to a `ClusterName` tag and the event rule name is now a prefix. This guarantees that users can have multiple instances of Karpenter withe their respective event rules/SQS queue without name collisions, while also still being able to identify which queues and event rules belong to which cluster. + +## Additional changes + +### Added + +- Support for setting `preserve` as well as `most_recent` on addons. + - `preserve` indicates if you want to preserve the created resources when deleting the EKS add-on + - `most_recent` indicates if you want to use the most recent revision of the add-on or the default version (default) +- Support for setting default node security group rules for common access patterns required: + - Egress all for `0.0.0.0/0`/`::/0` + - Ingress from cluster security group for 8443/TCP and 9443/TCP for common applications such as ALB Ingress Controller, Karpenter, OPA Gatekeeper, etc. These are commonly used as webhook ports for validating and mutating webhooks + +### Modified + +- `cluster_security_group_additional_rules` and `node_security_group_additional_rules` have been modified to use `lookup()` instead of `try()` to avoid the well known issue of [unkown values within a `for_each` loop](https://github.com/hashicorp/terraform/issues/4149) +- Default cluster security group rules have removed egress rules for TCP/443 and TCP/10250 to node groups since the cluster primary security group includes a default rule for ALL to `0.0.0.0/0`/`::/0` +- Default node security group rules have removed egress rules have been removed since the default security group settings have egress rule for ALL to `0.0.0.0/0`/`::/0` +- `block_device_mappings` previously required a map of maps but has since changed to an array of maps. Users can remove the outer key for each block device mapping and replace the outermost map `{}` with an array `[]`. There are no state changes required for this change. +- `create_kms_key` previously defaulted to `false` and now defaults to `true`. Clusters created with this module now default to enabling secret encryption by default with a customer managed KMS key created by this module +- `cluster_encryption_config` previously used a type of `list(any)` and now uses a type of `any` -> users can simply remove the outer `[`...`]` brackets on `v19.x` + - `cluster_encryption_config` previously defaulted to `[]` and now defaults to `{resources = ["secrets"]}` to encrypt secrets by default +- `cluster_endpoint_public_access` previously defaulted to `true` and now defaults to `false`. Clusters created with this module now default to private only access to the cluster endpoint + - `cluster_endpoint_private_access` previously defaulted to `false` and now defaults to `true` +- The addon configuration now sets `"OVERWRITE"` as the default value for `resolve_conflicts` to ease addon upgrade management. Users can opt out of this by instead setting `"NONE"` as the value for `resolve_conflicts` +- The `kms` module used has been updated from `v1.0.2` to `v1.1.0` - no material changes other than updated to latest +- The default value for EKS managed node group `update_config` has been updated to the recommended `{ max_unavailable_percentage = 33 }` +- The default value for the self-managed node group `instance_refresh` has been updated to the recommended: + ```hcl + { + strategy = "Rolling" + preferences = { + min_healthy_percentage = 66 + } + } + ``` + +### Removed + +- Remove all references of `aws_default_tags` to avoid update conflicts; this is the responsibility of the provider and should be handled at the provider level + - https://github.com/terraform-aws-modules/terraform-aws-eks/issues?q=is%3Aissue+default_tags+is%3Aclosed + - https://github.com/terraform-aws-modules/terraform-aws-eks/pulls?q=is%3Apr+default_tags+is%3Aclosed + +### Variable and output changes + +1. Removed variables: + + - `node_security_group_ntp_ipv4_cidr_block` - default security group settings have egress rule for ALL to `0.0.0.0/0`/`::/0` + - `node_security_group_ntp_ipv6_cidr_block` - default security group settings have egress rule for ALL to `0.0.0.0/0`/`::/0` + + - Self managed node groups: + - `create_security_group` + - `security_group_name` + - `security_group_use_name_prefix` + - `security_group_description` + - `security_group_rules` + - `security_group_tags` + - `cluster_security_group_id` + - `vpc_id` + - EKS managed node groups: + - `create_security_group` + - `security_group_name` + - `security_group_use_name_prefix` + - `security_group_description` + - `security_group_rules` + - `security_group_tags` + - `cluster_security_group_id` + - `vpc_id` + +2. Renamed variables: + + - N/A + +3. Added variables: + + - `provision_on_outpost`for Outposts support + - `outpost_config` for Outposts support + - `cluster_addons_timeouts` for setting a common set of timeouts for all addons (unless a specific value is provided within the addon configuration) + - `service_ipv6_cidr` for setting the IPv6 CIDR block for the Kubernetes service addresses + - `node_security_group_enable_recommended_rules` for enabling recommended node security group rules for common access patterns + + - Self managed node groups: + - `launch_template_id` for use when using an existing/externally created launch template (Ref: https://github.com/terraform-aws-modules/terraform-aws-autoscaling/pull/204) + - `maintenance_options` + - `private_dns_name_options` + - `instance_requirements` + - `context` + - `default_instance_warmup` + - `force_delete_warm_pool` + - EKS managed node groups: + - `use_custom_launch_template` was added to better clarify how users can switch betweeen a custom launch template or the default launch template provided by the EKS managed node group. Previously, to achieve this same functionality of using the default launch template, users needed to set `create_launch_template = false` and `launch_template_name = ""` which is not very intuitive. + - `launch_template_id` for use when using an existing/externally created launch template (Ref: https://github.com/terraform-aws-modules/terraform-aws-autoscaling/pull/204) + - `maintenance_options` + - `private_dns_name_options` + - +4. Removed outputs: + + - Self managed node groups: + - `security_group_arn` + - `security_group_id` + - EKS managed node groups: + - `security_group_arn` + - `security_group_id` + +5. Renamed outputs: + + - `cluster_id` is not renamed but the value it returns is now different. For standard EKS clusters created in the AWS cloud, the value returned at the time of this writing is `null`/empty. For local EKS clusters created on Outposts, the value returned will look like a UUID/GUID. Users should switch all instances of `cluster_id` to use `cluster_name` before upgrading to v19. [Reference](https://github.com/hashicorp/terraform-provider-aws/issues/27560) + +6. Added outputs: + + - `cluster_name` - The `cluster_id` currently set by the AWS provider is actually the cluster name, but in the future this will change and there will be a distinction between the `cluster_name` and `clsuter_id`. [Reference](https://github.com/hashicorp/terraform-provider-aws/issues/27560) + +## Upgrade Migrations + +1. Before upgrading your module definition to `v19.x`, please see below for both EKS managed node group(s) and self-managed node groups and removing the node group(s) security group prior to upgrading. + +### Self Managed Node Groups + +Self managed node groups on `v18.x` by default create a security group that does not specify any rules. In `v19.x`, this security group has been removed due to the predominant lack of usage (most users rely on the the shared node security group). While still using version `v18.x` of your module definition, remove this security group from your node groups by setting `create_security_group = false`. + +- If you are currently utilizing this security group, it is recommended to create an additional security group that matches the rules/settings of the security group created by the node group, and specify that security group ID in `vpc_security_group_ids`. Once this is in place, you can proceed with the original security group removal. +- For most users, the security group is not used and can be safely removed. However, deployed instances will have the security group attached to nodes and require the security group to be disassociated before the security group can be deleted. Because instances are deployed via autoscaling groups, we cannot simply remove the security group from code and have those changes reflected on the instances. Instead, we have to update the code and then trigger the autoscaling groups to cycle the instances deployed so that new instances are provisioned without the security group attached. You can utilize the `instance_refresh` parameter of Autoscaling groups to force nodes to re-deploy when removing the security group since changes to launch templates automatically trigger an instance refresh. An example configuration is provided below. + - Add the following to either/or `self_managed_node_group_defaults` or the individual self-managed node group definitions: + ```hcl + create_security_group = false + instance_refresh = { + strategy = "Rolling" + preferences = { + min_healthy_percentage = 66 + } + } + ``` +- It is recommended to use the `aws-node-termination-handler` while performing this update. Please refer to the [`irsa-autoscale-refresh` example](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/20af82846b4a1f23f3787a8c455f39c0b6164d80/examples/irsa_autoscale_refresh/charts.tf#L86) for usage. This will ensure that pods are safely evicted in a controlled manner to avoid service disruptions. +- Once the necessary configurations are in place, you can apply the changes which will: + 1. Create a new launch template (version) without the self-managed node group security group + 2. Replace instances based on the `instance_refresh` configuration settings + 3. New instances will launch without the self-managed node group security group, and prior instances will be terminated + 4. Once the self-managed node group has cycled, the security group will be deleted + +#### EKS Managed Node Groups + +EKS managed node groups on `v18.x` by default create a security group that does not specify any rules. In `v19.x`, this security group has been removed due to the predominant lack of usage (most users rely on the the shared node security group). While still using version `v18.x` of your module definition, remove this security group from your node groups by setting `create_security_group = false`. + +- If you are currently utilizing this security group, it is recommended to create an additional security group that matches the rules/settings of the security group created by the node group, and specify that security group ID in `vpc_security_group_ids`. Once this is in place, you can proceed with the original security group removal. +- EKS managed node groups rollout changes using a [rolling update strategy](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-update-behavior.html) that can be influenced through `update_config`. No additional changes are required for removing the the security group created by node groups (unlike self-managed node groups which should utilize the `instance_refresh` setting of Autoscaling groups). +- Once `create_security_group = false` has been set, you can apply the changes which will: + 1. Create a new launch template (version) without the EKS managed node group security group + 2. Replace instances based on the `update_config` configuration settings + 3. New instances will launch without the EKS managed node group security group, and prior instances will be terminated + 4. Once the EKS managed node group has cycled, the security group will be deleted + +2. Once the node group security group(s) have been removed, you can update your module definition to specify the `v19.x` version of the module +3. Run `terraform init -upgrade=true` to update your configuration and pull in the v19 changes +4. Using the documentation provided above, update your module definition to reflect the changes in the module from `v18.x` to `v19.x`. You can utilize `terraform plan` as you go to help highlight any changes that you wish to make. See below for `terraform state mv ...` commands related to the use of `iam_role_additional_policies`. If you are not providing any values to these variables, you can skip this section. +5. Once you are satisifed with the changes and the `terraform plan` output, you can apply the changes to sync your infrastructure with the updated module definition (or vice versa). + +### Diff of Before (v18.x) vs After (v19.x) + +```diff + module "eks" { + source = "terraform-aws-modules/eks/aws" +- version = "~> 18.0" ++ version = "~> 19.0" + + cluster_name = local.name ++ cluster_endpoint_public_access = true +- cluster_endpoint_private_access = true # now the default + + cluster_addons = { +- resolve_conflicts = "OVERWRITE" # now the default ++ preserve = true ++ most_recent = true + ++ timeouts = { ++ create = "25m" ++ delete = "10m" + } + kube-proxy = {} + vpc-cni = { +- resolve_conflicts = "OVERWRITE" # now the default + } + } + + # Encryption key + create_kms_key = true +- cluster_encryption_config = [{ +- resources = ["secrets"] +- }] ++ cluster_encryption_config = { ++ resources = ["secrets"] ++ } + kms_key_deletion_window_in_days = 7 + enable_kms_key_rotation = true + +- iam_role_additional_policies = [aws_iam_policy.additional.arn] ++ iam_role_additional_policies = { ++ additional = aws_iam_policy.additional.arn ++ } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + control_plane_subnet_ids = module.vpc.intra_subnets + + # Extend node-to-node security group rules +- node_security_group_ntp_ipv4_cidr_block = ["169.254.169.123/32"] # now the default + node_security_group_additional_rules = { +- ingress_self_ephemeral = { +- description = "Node to node ephemeral ports" +- protocol = "tcp" +- from_port = 0 +- to_port = 0 +- type = "ingress" +- self = true +- } +- egress_all = { +- description = "Node all egress" +- protocol = "-1" +- from_port = 0 +- to_port = 0 +- type = "egress" +- cidr_blocks = ["0.0.0.0/0"] +- ipv6_cidr_blocks = ["::/0"] +- } + } + + # Self Managed Node Group(s) + self_managed_node_group_defaults = { + vpc_security_group_ids = [aws_security_group.additional.id] +- iam_role_additional_policies = [aws_iam_policy.additional.arn] ++ iam_role_additional_policies = { ++ additional = aws_iam_policy.additional.arn ++ } + } + + self_managed_node_groups = { + spot = { + instance_type = "m5.large" + instance_market_options = { + market_type = "spot" + } + + pre_bootstrap_user_data = <<-EOT + echo "foo" + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + post_bootstrap_user_data = <<-EOT + cd /tmp + sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm + sudo systemctl enable amazon-ssm-agent + sudo systemctl start amazon-ssm-agent + EOT + +- create_security_group = true +- security_group_name = "eks-managed-node-group-complete-example" +- security_group_use_name_prefix = false +- security_group_description = "EKS managed node group complete example security group" +- security_group_rules = {} +- security_group_tags = {} + } + } + + # EKS Managed Node Group(s) + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] + + attach_cluster_primary_security_group = true + vpc_security_group_ids = [aws_security_group.additional.id] +- iam_role_additional_policies = [aws_iam_policy.additional.arn] ++ iam_role_additional_policies = { ++ additional = aws_iam_policy.additional.arn ++ } + } + + eks_managed_node_groups = { + blue = {} + green = { + min_size = 1 + max_size = 10 + desired_size = 1 + + instance_types = ["t3.large"] + capacity_type = "SPOT" + labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + taints = { + dedicated = { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + } + + update_config = { + max_unavailable_percentage = 33 # or set `max_unavailable` + } + +- create_security_group = true +- security_group_name = "eks-managed-node-group-complete-example" +- security_group_use_name_prefix = false +- security_group_description = "EKS managed node group complete example security group" +- security_group_rules = {} +- security_group_tags = {} + + tags = { + ExtraTag = "example" + } + } + } + + # Fargate Profile(s) + fargate_profile_defaults = { +- iam_role_additional_policies = [aws_iam_policy.additional.arn] ++ iam_role_additional_policies = { ++ additional = aws_iam_policy.additional.arn ++ } + } + + fargate_profiles = { + default = { + name = "default" + selectors = [ + { + namespace = "kube-system" + labels = { + k8s-app = "kube-dns" + } + }, + { + namespace = "default" + } + ] + + tags = { + Owner = "test" + } + + timeouts = { + create = "20m" + delete = "20m" + } + } + } + + # OIDC Identity provider + cluster_identity_providers = { + sts = { + client_id = "sts.amazonaws.com" + } + } + + # aws-auth configmap + manage_aws_auth_configmap = true + + aws_auth_node_iam_role_arns_non_windows = [ + module.eks_managed_node_group.iam_role_arn, + module.self_managed_node_group.iam_role_arn, + ] + aws_auth_fargate_profile_pod_execution_role_arns = [ + module.fargate_profile.fargate_profile_pod_execution_role_arn + ] + + aws_auth_roles = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] + + aws_auth_users = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] + + aws_auth_accounts = [ + "777777777777", + "888888888888", + ] + + tags = local.tags +} +``` + +## Terraform State Moves + +The following Terraform state move commands are optional but recommended if you are providing additional IAM policies that are to be attached to IAM roles created by this module (cluster IAM role, node group IAM role, Fargate profile IAM role). Because the resources affected are `aws_iam_role_policy_attachment`, in theory you could get away with simply applying the configuration and letting Terraform detach and re-attach the policies. However, during this brief period of update, you could experience permission failures as the policy is detached and re-attached and therefore the state move route is recommended. + +Where `""` is specified, this should be replaced with the full ARN of the policy, and `""` should be replaced with the key used in the `iam_role_additional_policies` map for the associated policy. For example, if you have the following`v19.x` configuration: + +```hcl + ... + # This is demonstrating the cluster IAM role addtional policies + iam_role_additional_policies = { + additional = aws_iam_policy.additional.arn + } + ... +``` + +The associated state move command would look similar to (albeit with your correct policy ARN): + +```sh +terraform state mv 'module.eks.aws_iam_role_policy_attachment.this["arn:aws:iam::111111111111:policy/ex-complete-additional"]' 'module.eks.aws_iam_role_policy_attachment.additional["additional"]' +``` + +If you are not providing any additional IAM policies, no actions are required. + +### Cluster IAM Role + +Repeat for each policy provided in `iam_role_additional_policies`: + +```sh +terraform state mv 'module.eks.aws_iam_role_policy_attachment.this[""]' 'module.eks.aws_iam_role_policy_attachment.additional[""]' +``` + +### EKS Managed Node Group IAM Role + +Where `""` is the key used in the `eks_managed_node_groups` map for the associated node group. Repeat for each policy provided in `iam_role_additional_policies` in either/or `eks_managed_node_group_defaults` or the individual node group definitions: + +```sh +terraform state mv 'module.eks.module.eks_managed_node_group[""].aws_iam_role_policy_attachment.this[""]' 'module.eks.module.eks_managed_node_group[""].aws_iam_role_policy_attachment.additional[""]' +``` + +### Self-Managed Node Group IAM Role + +Where `""` is the key used in the `self_managed_node_groups` map for the associated node group. Repeat for each policy provided in `iam_role_additional_policies` in either/or `self_managed_node_group_defaults` or the individual node group definitions: + +```sh +terraform state mv 'module.eks.module.self_managed_node_group[""].aws_iam_role_policy_attachment.this[""]' 'module.eks.module.self_managed_node_group[""].aws_iam_role_policy_attachment.additional[""]' +``` + +### Fargate Profile IAM Role + +Where `""` is the key used in the `fargate_profiles` map for the associated profile. Repeat for each policy provided in `iam_role_additional_policies` in either/or `fargate_profile_defaults` or the individual profile definitions: + +```sh +terraform state mv 'module.eks.module.fargate_profile[""].aws_iam_role_policy_attachment.this[""]' 'module.eks.module.fargate_profile[""].aws_iam_role_policy_attachment.additional[""]' +``` diff --git a/docs/compute_resources.md b/docs/compute_resources.md index d90cf5f0a8..05d5ea1dd8 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -18,8 +18,7 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ ```hcl eks_managed_node_groups = { default = { - create_launch_template = false - launch_template_name = "" + use_custom_launch_template = false } } ``` @@ -29,8 +28,7 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ ```hcl eks_managed_node_groups = { bottlerocket_default = { - create_launch_template = false - launch_template_name = "" + use_custom_launch_template = false ami_type = "BOTTLEROCKET_x86_64" platform = "bottlerocket" @@ -45,15 +43,15 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ prepend_userdata = { # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 pre_bootstrap_user_data = <<-EOT - #!/bin/bash - set -ex - cat <<-EOF > /etc/profile.d/bootstrap.sh - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - export KUBELET_EXTRA_ARGS="--max-pods=110" - EOF - # Source extra environment variables in bootstrap script - sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + export KUBELET_EXTRA_ARGS="--max-pods=110" + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh EOT } } @@ -68,9 +66,9 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ platform = "bottlerocket" bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" + # extra args added + [settings.kernel] + lockdown = "integrity" EOT } } @@ -116,17 +114,17 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ enable_bootstrap_user_data = true # this will get added to the template bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" + # extra args added + [settings.kernel] + lockdown = "integrity" - [settings.kubernetes.node-labels] - "label1" = "foo" - "label2" = "bar" + [settings.kubernetes.node-labels] + "label1" = "foo" + "label2" = "bar" - [settings.kubernetes.node-taints] - "dedicated" = "experimental:PreferNoSchedule" - "special" = "true:NoSchedule" + [settings.kubernetes.node-taints] + "dedicated" = "experimental:PreferNoSchedule" + "special" = "true:NoSchedule" EOT } } diff --git a/docs/faq.md b/docs/faq.md index c1ab564ee0..c53b027dca 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,7 +1,6 @@ # Frequently Asked Questions - [I received an error: `expect exactly one securityGroup tagged with kubernetes.io/cluster/ ...`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#i-received-an-error-expect-exactly-one-securitygroup-tagged-with-kubernetesioclustername-) -- [I received an error: `Error: Invalid for_each argument ...`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#i-received-an-error-error-invalid-for_each-argument-) - [Why are nodes not being registered?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-nodes-not-being-registered) - [Why are there no changes when a node group's `desired_size` is modified?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-there-no-changes-when-a-node-groups-desired_size-is-modified) - [How can I deploy Windows based nodes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-can-i-deploy-windows-based-nodes) @@ -48,41 +47,6 @@ By default, EKS creates a cluster primary security group that is created outside In theory, if you are attaching the cluster primary security group, you shouldn't need to use the shared node security group created by the module. However, this is left up to users to decide for their requirements and use case. -### I received an error: `Error: Invalid for_each argument ...` - -Users may encounter an error such as `Error: Invalid for_each argument - The "for_each" value depends on resource attributes that cannot be determined until apply, so Terraform cannot predict how many instances will be created. To work around this, use the -target argument to first apply ...` - -This error is due to an upstream issue with [Terraform core](https://github.com/hashicorp/terraform/issues/4149). There are two potential options you can take to help mitigate this issue: - -1. Create the dependent resources before the cluster => `terraform apply -target ` and then `terraform apply` for the cluster (or other similar means to just ensure the referenced resources exist before creating the cluster) - -- Note: this is the route users will have to take for adding additional security groups to nodes since there isn't a separate "security group attachment" resource - -2. For additional IAM policies, users can attach the policies outside of the cluster definition as demonstrated below - -```hcl -resource "aws_iam_role_policy_attachment" "additional" { - for_each = module.eks.eks_managed_node_groups - # you could also do the following or any combination: - # for_each = merge( - # module.eks.eks_managed_node_groups, - # module.eks.self_managed_node_group, - # module.eks.fargate_profile, - # ) - - # This policy does not have to exist at the time of cluster creation. Terraform can - # deduce the proper order of its creation to avoid errors during creation - policy_arn = aws_iam_policy.node_additional.arn - role = each.value.iam_role_name -} -``` - -TL;DR - Terraform resource passed into the modules map definition _must_ be known before you can apply the EKS module. The variables this potentially affects are: - -- `cluster_security_group_additional_rules` (i.e. - referencing an external security group resource in a rule) -- `node_security_group_additional_rules` (i.e. - referencing an external security group resource in a rule) -- `iam_role_additional_policies` (i.e. - referencing an external policy resource) - ### Why are nodes not being registered? Nodes not being able to register with the EKS control plane is generally due to networking mis-configurations. diff --git a/docs/network_connectivity.md b/docs/network_connectivity.md index 67805aa77c..9d38fc130d 100644 --- a/docs/network_connectivity.md +++ b/docs/network_connectivity.md @@ -20,8 +20,7 @@ Please refer to the [AWS documentation](https://docs.aws.amazon.com/eks/latest/u - Lastly, users are able to opt in to attaching the primary security group automatically created by the EKS service by setting `attach_cluster_primary_security_group` = `true` from the root module for the respective node group (or set it within the node group defaults). This security group is not managed by the module; it is created by the EKS service. It permits all traffic within the domain of the security group as well as all egress traffic to the internet. - Node Group Security Group(s) - - Each node group (EKS Managed Node Group and Self Managed Node Group) by default creates its own security group. By default, this security group does not contain any additional security group rules. It is merely an "empty container" that offers users the ability to opt into any addition inbound our outbound rules as necessary - - Users also have the option to supply their own, and/or additional, externally created security group(s) to the node group as well via the `vpc_security_group_ids` variable + - Users have the option to assign their own externally created security group(s) to the node group via the `vpc_security_group_ids` variable See the example snippet below which adds additional security group rules to the cluster security group as well as the shared node security group (for node-to-node access). Users can use this extensibility to open up network access as they see fit using the security groups provided by the module: diff --git a/docs/user_data.md b/docs/user_data.md index e5c247b798..4ce9a13ac4 100644 --- a/docs/user_data.md +++ b/docs/user_data.md @@ -55,15 +55,15 @@ Since the EKS Managed Node Group service provides the necessary bootstrap user d ```hcl # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 pre_bootstrap_user_data = <<-EOT - #!/bin/bash - set -ex - cat <<-EOF > /etc/profile.d/bootstrap.sh - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - export KUBELET_EXTRA_ARGS="--max-pods=110" - EOF - # Source extra environment variables in bootstrap script - sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + export KUBELET_EXTRA_ARGS="--max-pods=110" + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh EOT ``` diff --git a/examples/complete/README.md b/examples/complete/README.md index 66f3e8fdf1..6f5edbdb41 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -33,15 +33,15 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.72 | +| [aws](#provider\_aws) | >= 4.45 | ## Modules @@ -61,7 +61,9 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| +| [aws_iam_policy.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | ## Inputs @@ -81,9 +83,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | -| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 04817fa78b..a040d750f2 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -14,10 +14,15 @@ provider "kubernetes" { } } +data "aws_availability_zones" "available" {} + locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" region = "eu-west-1" + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + tags = { Example = local.name GithubRepo = "terraform-aws-eks" @@ -32,46 +37,56 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true + cluster_name = local.name + cluster_endpoint_public_access = true cluster_addons = { coredns = { - resolve_conflicts = "OVERWRITE" + preserve = true + most_recent = true + + timeouts = { + create = "25m" + delete = "10m" + } + } + kube-proxy = { + most_recent = true } - kube-proxy = {} vpc-cni = { - resolve_conflicts = "OVERWRITE" + most_recent = true } } # Encryption key create_kms_key = true - cluster_encryption_config = [{ + cluster_encryption_config = { resources = ["secrets"] - }] + } kms_key_deletion_window_in_days = 7 enable_kms_key_rotation = true + iam_role_additional_policies = { + additional = aws_iam_policy.additional.arn + } + vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets # Extend cluster security group rules cluster_security_group_additional_rules = { - egress_nodes_ephemeral_ports_tcp = { - description = "To node 1025-65535" + ingress_nodes_ephemeral_ports_tcp = { + description = "Nodes on ephemeral ports" protocol = "tcp" from_port = 1025 to_port = 65535 - type = "egress" + type = "ingress" source_node_security_group = true } } # Extend node-to-node security group rules - node_security_group_ntp_ipv4_cidr_block = ["169.254.169.123/32"] node_security_group_additional_rules = { ingress_self_all = { description = "Node to node all ports/protocols" @@ -81,21 +96,21 @@ module "eks" { type = "ingress" self = true } - egress_all = { - description = "Node all egress" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "egress" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] - } } # Self Managed Node Group(s) self_managed_node_group_defaults = { - vpc_security_group_ids = [aws_security_group.additional.id] - iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"] + vpc_security_group_ids = [aws_security_group.additional.id] + iam_role_additional_policies = { + additional = aws_iam_policy.additional.arn + } + + instance_refresh = { + strategy = "Rolling" + preferences = { + min_healthy_percentage = 66 + } + } } self_managed_node_groups = { @@ -106,17 +121,17 @@ module "eks" { } pre_bootstrap_user_data = <<-EOT - echo "foo" - export FOO=bar + echo "foo" + export FOO=bar EOT bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" post_bootstrap_user_data = <<-EOT - cd /tmp - sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm - sudo systemctl enable amazon-ssm-agent - sudo systemctl start amazon-ssm-agent + cd /tmp + sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm + sudo systemctl enable amazon-ssm-agent + sudo systemctl start amazon-ssm-agent EOT } } @@ -128,6 +143,9 @@ module "eks" { attach_cluster_primary_security_group = true vpc_security_group_ids = [aws_security_group.additional.id] + iam_role_additional_policies = { + additional = aws_iam_policy.additional.arn + } } eks_managed_node_groups = { @@ -154,7 +172,7 @@ module "eks" { } update_config = { - max_unavailable_percentage = 50 # or set `max_unavailable` + max_unavailable_percentage = 33 # or set `max_unavailable` } tags = { @@ -270,7 +288,6 @@ module "eks_managed_node_group" { cluster_name = module.eks.cluster_name cluster_version = module.eks.cluster_version - vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id vpc_security_group_ids = [ @@ -305,7 +322,6 @@ module "self_managed_node_group" { instance_type = "m5.large" - vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets vpc_security_group_ids = [ module.eks.cluster_primary_security_group_id, @@ -366,12 +382,12 @@ module "vpc" { version = "~> 3.0" name = local.name - cidr = "10.0.0.0/16" + cidr = local.vpc_cidr - azs = ["${local.region}a", "${local.region}b", "${local.region}c"] - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - intra_subnets = ["10.0.7.0/28", "10.0.7.16/28", "10.0.7.32/28"] + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] enable_nat_gateway = true single_nat_gateway = true @@ -382,13 +398,11 @@ module "vpc" { create_flow_log_cloudwatch_log_group = true public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = 1 + "kubernetes.io/role/elb" = 1 } private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = 1 + "kubernetes.io/role/internal-elb" = 1 } tags = local.tags @@ -409,5 +423,22 @@ resource "aws_security_group" "additional" { ] } - tags = local.tags + tags = merge(local.tags, { Name = "${local.name}-additional" }) +} + +resource "aws_iam_policy" "additional" { + name = "${local.name}-additional" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:Describe*", + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) } diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index f94c629541..c1020f3333 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -17,16 +17,16 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } -output "cluster_name" { - description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" - value = module.eks.cluster_name -} - output "cluster_id" { - description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" value = module.eks.cluster_id } +output "cluster_name" { + description = "The name of the EKS cluster" + value = module.eks.cluster_name +} + output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster for the OpenID Connect identity provider" value = module.eks.cluster_oidc_issuer_url diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index 6d6dc45be6..f2f8625d4b 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72" + version = ">= 4.45" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 449777a52a..42fb41e82c 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -57,45 +57,37 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | -| [tls](#requirement\_tls) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.72 | -| [tls](#provider\_tls) | >= 3.0 | +| [aws](#provider\_aws) | >= 4.45 | ## Modules | Name | Source | Version | |------|--------|---------| +| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 1.1 | | [eks](#module\_eks) | ../.. | n/a | +| [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | -| [vpc\_cni\_irsa](#module\_vpc\_cni\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.12 | +| [vpc\_cni\_irsa](#module\_vpc\_cni\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.0 | ## Resources | Name | Type | |------|------| -| [aws_autoscaling_group_tag.cluster_autoscaler_label_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group_tag) | resource | | [aws_iam_policy.node_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_key_pair.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | -| [aws_kms_key.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | -| [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | -| [aws_launch_template.external](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | -| [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.remote_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_ami.eks_default_arm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | ## Inputs @@ -115,9 +107,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | -| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 050a508810..ebf39cdd16 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -14,11 +14,17 @@ provider "kubernetes" { } } +data "aws_caller_identity" "current" {} +data "aws_availability_zones" "available" {} + locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" cluster_version = "1.24" region = "eu-west-1" + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + tags = { Example = local.name GithubRepo = "terraform-aws-eks" @@ -26,8 +32,6 @@ locals { } } -data "aws_caller_identity" "current" {} - ################################################################################ # EKS Module ################################################################################ @@ -35,10 +39,9 @@ data "aws_caller_identity" "current" {} module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true # IPV6 cluster_ip_family = "ipv6" @@ -53,66 +56,23 @@ module "eks" { cluster_addons = { coredns = { - resolve_conflicts = "OVERWRITE" + most_recent = true + } + kube-proxy = { + most_recent = true } - kube-proxy = {} vpc-cni = { - resolve_conflicts = "OVERWRITE" + most_recent = true service_account_role_arn = module.vpc_cni_irsa.iam_role_arn } } - cluster_encryption_config = [{ - provider_key_arn = aws_kms_key.eks.arn - resources = ["secrets"] - }] - - cluster_tags = { - # This should not affect the name of the cluster primary security group - # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2006 - # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2008 - Name = local.name - } - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + control_plane_subnet_ids = module.vpc.intra_subnets manage_aws_auth_configmap = true - # Extend cluster security group rules - cluster_security_group_additional_rules = { - egress_nodes_ephemeral_ports_tcp = { - description = "To node 1025-65535" - protocol = "tcp" - from_port = 1025 - to_port = 65535 - type = "egress" - source_node_security_group = true - } - } - - # Extend node-to-node security group rules - node_security_group_ntp_ipv6_cidr_block = ["fd00:ec2::123/128"] - node_security_group_additional_rules = { - ingress_self_all = { - description = "Node to node all ports/protocols" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "ingress" - self = true - } - egress_all = { - description = "Node all egress" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "egress" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] - } - } - eks_managed_node_group_defaults = { ami_type = "AL2_x86_64" instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] @@ -130,14 +90,13 @@ module "eks" { default_node_group = { # By default, the module creates a launch template to ensure tags are propagated to instances, etc., # so we need to disable it to use the default template provided by the AWS EKS managed node group service - create_launch_template = false - launch_template_name = "" + use_custom_launch_template = false disk_size = 50 # Remote access cannot be specified with a launch template remote_access = { - ec2_ssh_key = aws_key_pair.this.key_name + ec2_ssh_key = module.key_pair.key_pair_name source_security_group_ids = [aws_security_group.remote_access.id] } } @@ -146,8 +105,7 @@ module "eks" { bottlerocket_default = { # By default, the module creates a launch template to ensure tags are propagated to instances, etc., # so we need to disable it to use the default template provided by the AWS EKS managed node group service - create_launch_template = false - launch_template_name = "" + use_custom_launch_template = false ami_type = "BOTTLEROCKET_x86_64" platform = "bottlerocket" @@ -158,11 +116,11 @@ module "eks" { ami_type = "BOTTLEROCKET_x86_64" platform = "bottlerocket" - # this will get added to what AWS provides + # This will get added to what AWS provides bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" + # extra args added + [settings.kernel] + lockdown = "integrity" EOT } @@ -172,31 +130,35 @@ module "eks" { ami_id = data.aws_ami.eks_default_bottlerocket.image_id platform = "bottlerocket" - # use module user data template to boostrap + # Use module user data template to boostrap enable_bootstrap_user_data = true - # this will get added to the template + # This will get added to the template bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" - - [settings.kubernetes.node-labels] - "label1" = "foo" - "label2" = "bar" - - [settings.kubernetes.node-taints] - "dedicated" = "experimental:PreferNoSchedule" - "special" = "true:NoSchedule" + # The admin host container provides SSH access and runs with "superpowers". + # It is disabled by default, but can be disabled explicitly. + [settings.host-containers.admin] + enabled = false + + # The control host container provides out-of-band access via SSM. + # It is enabled by default, and can be disabled if you do not expect to use SSM. + # This could leave you with no way to access the API and change settings on an existing node! + [settings.host-containers.control] + enabled = true + + # extra args added + [settings.kernel] + lockdown = "integrity" + + [settings.kubernetes.node-labels] + label1 = "foo" + label2 = "bar" + + [settings.kubernetes.node-taints] + dedicated = "experimental:PreferNoSchedule" + special = "true:NoSchedule" EOT } - # Use existing/external launch template - external_lt = { - create_launch_template = false - launch_template_name = aws_launch_template.external.name - launch_template_version = aws_launch_template.external.default_version - } - # Use a custom AMI custom_ami = { ami_type = "AL2_ARM_64" @@ -219,15 +181,15 @@ module "eks" { # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 pre_bootstrap_user_data = <<-EOT - #!/bin/bash - set -ex - cat <<-EOF > /etc/profile.d/bootstrap.sh - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - export KUBELET_EXTRA_ARGS="--max-pods=110" - EOF - # Source extra environment variables in bootstrap script - sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + export KUBELET_EXTRA_ARGS="--max-pods=110" + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh EOT } @@ -247,12 +209,12 @@ module "eks" { bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false EOT post_bootstrap_user_data = <<-EOT - echo "you are free little kubelet!" + echo "you are free little kubelet!" EOT capacity_type = "SPOT" @@ -272,13 +234,12 @@ module "eks" { ] update_config = { - max_unavailable_percentage = 50 # or set `max_unavailable` + max_unavailable_percentage = 33 # or set `max_unavailable` } description = "EKS managed node group example launch template" ebs_optimized = true - vpc_security_group_ids = [aws_security_group.additional.id] disable_api_termination = false enable_monitoring = true @@ -291,7 +252,7 @@ module "eks" { iops = 3000 throughput = 150 encrypted = true - kms_key_id = aws_kms_key.ebs.arn + kms_key_id = module.ebs_kms_key.key_id delete_on_termination = true } } @@ -311,34 +272,9 @@ module "eks" { iam_role_tags = { Purpose = "Protector of the kubelet" } - iam_role_additional_policies = [ - "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" - ] - - create_security_group = true - security_group_name = "eks-managed-node-group-complete-example" - security_group_use_name_prefix = false - security_group_description = "EKS managed node group complete example security group" - security_group_rules = { - phoneOut = { - description = "Hello CloudFlare" - protocol = "udp" - from_port = 53 - to_port = 53 - type = "egress" - cidr_blocks = ["1.1.1.1/32"] - } - phoneHome = { - description = "Hello cluster" - protocol = "udp" - from_port = 53 - to_port = 53 - type = "egress" - source_cluster_security_group = true # bit of reflection lookup - } - } - security_group_tags = { - Purpose = "Protector of the kubelet" + iam_role_additional_policies = { + AmazonEC2ContainerRegistryReadOnly = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + additional = aws_iam_policy.node_additional.arn } tags = { @@ -350,18 +286,6 @@ module "eks" { tags = local.tags } -# References to resources that do not exist yet when creating a cluster will cause a plan failure due to https://github.com/hashicorp/terraform/issues/4149 -# There are two options users can take -# 1. Create the dependent resources before the cluster => `terraform apply -target and then `terraform apply` -# Note: this is the route users will have to take for adding additonal security groups to nodes since there isn't a separate "security group attachment" resource -# 2. For addtional IAM policies, users can attach the policies outside of the cluster definition as demonstrated below -resource "aws_iam_role_policy_attachment" "additional" { - for_each = module.eks.eks_managed_node_groups - - policy_arn = aws_iam_policy.node_additional.arn - role = each.value.iam_role_name -} - ################################################################################ # Supporting Resources ################################################################################ @@ -371,11 +295,12 @@ module "vpc" { version = "~> 3.0" name = local.name - cidr = "10.0.0.0/16" + cidr = local.vpc_cidr - azs = ["${local.region}a", "${local.region}b", "${local.region}c"] - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] enable_ipv6 = true assign_ipv6_address_on_creation = true @@ -393,13 +318,11 @@ module "vpc" { create_flow_log_cloudwatch_log_group = true public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = 1 + "kubernetes.io/role/elb" = 1 } private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = 1 + "kubernetes.io/role/internal-elb" = 1 } tags = local.tags @@ -407,7 +330,7 @@ module "vpc" { module "vpc_cni_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 4.12" + version = "~> 5.0" role_name_prefix = "VPC-CNI-IRSA" attach_vpc_cni_policy = true @@ -423,175 +346,35 @@ module "vpc_cni_irsa" { tags = local.tags } -resource "aws_security_group" "additional" { - name_prefix = "${local.name}-additional" - vpc_id = module.vpc.vpc_id +module "ebs_kms_key" { + source = "terraform-aws-modules/kms/aws" + version = "~> 1.1" - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = [ - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - ] - } + description = "Customer managed key to encrypt EKS managed node group volumes" - tags = local.tags -} + # Policy + key_administrators = [ + data.aws_caller_identity.current.arn + ] + key_service_users = [ + # required for the ASG to manage encrypted volumes for nodes + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", + # required for the cluster / persistentvolume-controller to create encrypted PVCs + module.eks.cluster_iam_role_arn, + ] -resource "aws_kms_key" "eks" { - description = "EKS Secret Encryption Key" - deletion_window_in_days = 7 - enable_key_rotation = true + # Aliases + aliases = ["eks/${local.name}/ebs"] tags = local.tags } -resource "aws_kms_key" "ebs" { - description = "Customer managed key to encrypt EKS managed node group volumes" - deletion_window_in_days = 7 - policy = data.aws_iam_policy_document.ebs.json -} - -# This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes -data "aws_iam_policy_document" "ebs" { - # Copy of default KMS policy that lets you manage it - statement { - sid = "Enable IAM User Permissions" - actions = ["kms:*"] - resources = ["*"] - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] - } - } - - # Required for EKS - statement { - sid = "Allow service-linked role use of the CMK" - actions = [ - "kms:Encrypt", - "kms:Decrypt", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:DescribeKey" - ] - resources = ["*"] - - principals { - type = "AWS" - identifiers = [ - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes - module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs - ] - } - } - - statement { - sid = "Allow attachment of persistent resources" - actions = ["kms:CreateGrant"] - resources = ["*"] - - principals { - type = "AWS" - identifiers = [ - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes - module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs - ] - } - - condition { - test = "Bool" - variable = "kms:GrantIsForAWSResource" - values = ["true"] - } - } -} - -# This is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) -# there are several more options one could set but you probably dont need to modify them -# you can take the default and add your custom AMI and/or custom tags -# -# Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, -# then the default user-data for bootstrapping a cluster is merged in the copy. - -resource "aws_launch_template" "external" { - name_prefix = "external-eks-ex-" - description = "EKS managed node group external launch template" - update_default_version = true - - block_device_mappings { - device_name = "/dev/xvda" - - ebs { - volume_size = 100 - volume_type = "gp2" - delete_on_termination = true - } - } - - monitoring { - enabled = true - } - - # Disabling due to https://github.com/hashicorp/terraform-provider-aws/issues/23766 - # network_interfaces { - # associate_public_ip_address = false - # delete_on_termination = true - # } - - # if you want to use a custom AMI - # image_id = var.ami_id - - # If you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then - # you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 - # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) - # user_data = base64encode(data.template_file.launch_template_userdata.rendered) - - tag_specifications { - resource_type = "instance" - - tags = { - Name = "external_lt" - CustomTag = "Instance custom tag" - } - } - - tag_specifications { - resource_type = "volume" +module "key_pair" { + source = "terraform-aws-modules/key-pair/aws" + version = "~> 2.0" - tags = { - CustomTag = "Volume custom tag" - } - } - - tag_specifications { - resource_type = "network-interface" - - tags = { - CustomTag = "EKS example" - } - } - - tags = { - CustomTag = "Launch template custom tag" - } - - lifecycle { - create_before_destroy = true - } -} - -resource "tls_private_key" "this" { - algorithm = "RSA" -} - -resource "aws_key_pair" "this" { - key_name_prefix = local.name - public_key = tls_private_key.this.public_key_openssh + key_name_prefix = local.name + create_private_key = true tags = local.tags } @@ -617,7 +400,7 @@ resource "aws_security_group" "remote_access" { ipv6_cidr_blocks = ["::/0"] } - tags = local.tags + tags = merge(local.tags, { Name = "${local.name}-remote" }) } resource "aws_iam_policy" "node_additional" { @@ -669,52 +452,3 @@ data "aws_ami" "eks_default_bottlerocket" { values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"] } } - -################################################################################ -# Tags for the ASG to support cluster-autoscaler scale up from 0 -################################################################################ - -locals { - - # We need to lookup K8s taint effect from the AWS API value - taint_effects = { - NO_SCHEDULE = "NoSchedule" - NO_EXECUTE = "NoExecute" - PREFER_NO_SCHEDULE = "PreferNoSchedule" - } - - cluster_autoscaler_label_tags = merge([ - for name, group in module.eks.eks_managed_node_groups : { - for label_name, label_value in coalesce(group.node_group_labels, {}) : "${name}|label|${label_name}" => { - autoscaling_group = group.node_group_autoscaling_group_names[0], - key = "k8s.io/cluster-autoscaler/node-template/label/${label_name}", - value = label_value, - } - } - ]...) - - cluster_autoscaler_taint_tags = merge([ - for name, group in module.eks.eks_managed_node_groups : { - for taint in coalesce(group.node_group_taints, []) : "${name}|taint|${taint.key}" => { - autoscaling_group = group.node_group_autoscaling_group_names[0], - key = "k8s.io/cluster-autoscaler/node-template/taint/${taint.key}" - value = "${taint.value}:${local.taint_effects[taint.effect]}" - } - } - ]...) - - cluster_autoscaler_asg_tags = merge(local.cluster_autoscaler_label_tags, local.cluster_autoscaler_taint_tags) -} - -resource "aws_autoscaling_group_tag" "cluster_autoscaler_label_tags" { - for_each = local.cluster_autoscaler_asg_tags - - autoscaling_group_name = each.value.autoscaling_group - - tag { - key = each.value.key - value = each.value.value - - propagate_at_launch = false - } -} diff --git a/examples/eks_managed_node_group/outputs.tf b/examples/eks_managed_node_group/outputs.tf index 67de532779..43334ecc0a 100644 --- a/examples/eks_managed_node_group/outputs.tf +++ b/examples/eks_managed_node_group/outputs.tf @@ -17,16 +17,16 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } -output "cluster_name" { - description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" - value = module.eks.cluster_name -} - output "cluster_id" { - description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" value = module.eks.cluster_id } +output "cluster_name" { + description = "The name of the EKS cluster" + value = module.eks.cluster_name +} + output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster for the OpenID Connect identity provider" value = module.eks.cluster_oidc_issuer_url diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index fde7af0f23..f2f8625d4b 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -1,14 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72" - } - tls = { - source = "hashicorp/tls" - version = ">= 3.0" + version = ">= 4.45" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 2f9cf7da1b..71a9ee7ad0 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -19,8 +19,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | | [helm](#requirement\_helm) | >= 2.7 | | [null](#requirement\_null) | >= 3.0 | @@ -28,7 +28,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.72 | +| [aws](#provider\_aws) | >= 4.45 | | [helm](#provider\_helm) | >= 2.7 | | [null](#provider\_null) | >= 3.0 | @@ -43,10 +43,11 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| -| [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_iam_policy.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [helm_release.coredns](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [null_resource.modify_kube_dns](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.remove_default_coredns_deployment](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | @@ -68,9 +69,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | -| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index 0bf15a7442..8eb0c934fe 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -16,11 +16,16 @@ provider "helm" { } } +data "aws_availability_zones" "available" {} + locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" cluster_version = "1.24" region = "eu-west-1" + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + tags = { Example = local.name GithubRepo = "terraform-aws-eks" @@ -35,28 +40,29 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true cluster_addons = { kube-proxy = {} vpc-cni = {} } - cluster_encryption_config = [{ - provider_key_arn = aws_kms_key.eks.arn - resources = ["secrets"] - }] - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + control_plane_subnet_ids = module.vpc.intra_subnets # Fargate profiles use the cluster primary security group so these are not utilized create_cluster_security_group = false create_node_security_group = false + fargate_profile_defaults = { + iam_role_additional_policies = { + additional = aws_iam_policy.additional.arn + } + } + fargate_profiles = { example = { name = "example" @@ -231,11 +237,12 @@ module "vpc" { version = "~> 3.0" name = local.name - cidr = "10.0.0.0/16" + cidr = local.vpc_cidr - azs = ["${local.region}a", "${local.region}b", "${local.region}c"] - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] enable_nat_gateway = true single_nat_gateway = true @@ -246,22 +253,29 @@ module "vpc" { create_flow_log_cloudwatch_log_group = true public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = 1 + "kubernetes.io/role/elb" = 1 } private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = 1 + "kubernetes.io/role/internal-elb" = 1 } tags = local.tags } -resource "aws_kms_key" "eks" { - description = "EKS Secret Encryption Key" - deletion_window_in_days = 7 - enable_key_rotation = true - - tags = local.tags +resource "aws_iam_policy" "additional" { + name = "${local.name}-additional" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:Describe*", + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) } diff --git a/examples/fargate_profile/outputs.tf b/examples/fargate_profile/outputs.tf index 67de532779..43334ecc0a 100644 --- a/examples/fargate_profile/outputs.tf +++ b/examples/fargate_profile/outputs.tf @@ -17,16 +17,16 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } -output "cluster_name" { - description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" - value = module.eks.cluster_name -} - output "cluster_id" { - description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" value = module.eks.cluster_id } +output "cluster_name" { + description = "The name of the EKS cluster" + value = module.eks.cluster_name +} + output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster for the OpenID Connect identity provider" value = module.eks.cluster_oidc_issuer_url diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index 5128bc99e3..69ef526bd9 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72" + version = ">= 4.45" } helm = { source = "hashicorp/helm" diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 7b865797a4..f5d4e2dcc3 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -51,9 +51,9 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | -| [helm](#requirement\_helm) | >= 2.4 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | +| [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 1.14 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | | [null](#requirement\_null) | >= 3.0 | @@ -62,8 +62,9 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.72 | -| [helm](#provider\_helm) | >= 2.4 | +| [aws](#provider\_aws) | >= 4.45 | +| [aws.virginia](#provider\_aws.virginia) | >= 4.45 | +| [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 1.14 | | [null](#provider\_null) | >= 3.0 | @@ -86,6 +87,8 @@ Note that this example may create resources which cost money. Run `terraform des | [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [null_resource.modify_kube_dns](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.remove_default_coredns_deployment](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | | [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | @@ -107,9 +110,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | -| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 3634132ee2..0a8725e466 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -2,6 +2,11 @@ provider "aws" { region = local.region } +provider "aws" { + region = "us-east-1" + alias = "virginia" +} + provider "kubernetes" { host = module.eks.cluster_endpoint cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) @@ -42,11 +47,19 @@ provider "kubectl" { } } +data "aws_availability_zones" "available" {} +data "aws_ecrpublic_authorization_token" "token" { + provider = aws.virginia +} + locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" cluster_version = "1.24" region = "eu-west-1" + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + tags = { Example = local.name GithubRepo = "terraform-aws-eks" @@ -61,17 +74,13 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true - # Fargate profiles use the cluster primary security group so these are not utilized - create_cluster_security_group = false - create_node_security_group = false + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + control_plane_subnet_ids = module.vpc.intra_subnets manage_aws_auth_configmap = true aws_auth_roles = [ @@ -114,7 +123,6 @@ module "eks" { # Karpenter ################################################################################ - module "karpenter" { source = "../../modules/karpenter" @@ -128,10 +136,12 @@ resource "helm_release" "karpenter" { namespace = "karpenter" create_namespace = true - name = "karpenter" - repository = "oci://public.ecr.aws/karpenter" - chart = "karpenter" - version = "v0.19.1" + name = "karpenter" + repository = "oci://public.ecr.aws/karpenter" + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password + chart = "karpenter" + version = "v0.19.3" set { name = "settings.aws.clusterName" @@ -367,24 +377,27 @@ module "vpc" { version = "~> 3.0" name = local.name - cidr = "10.0.0.0/16" + cidr = local.vpc_cidr - azs = ["${local.region}a", "${local.region}b", "${local.region}c"] - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] enable_nat_gateway = true single_nat_gateway = true enable_dns_hostnames = true + enable_flow_log = true + create_flow_log_cloudwatch_iam_role = true + create_flow_log_cloudwatch_log_group = true + public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = 1 + "kubernetes.io/role/elb" = 1 } private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = 1 + "kubernetes.io/role/internal-elb" = 1 # Tags subnets for Karpenter auto-discovery "karpenter.sh/discovery" = local.name } diff --git a/examples/karpenter/outputs.tf b/examples/karpenter/outputs.tf index 39d418bf37..f0ad50bd6a 100644 --- a/examples/karpenter/outputs.tf +++ b/examples/karpenter/outputs.tf @@ -17,16 +17,16 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } -output "cluster_name" { - description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" - value = module.eks.cluster_name -} - output "cluster_id" { - description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" value = module.eks.cluster_id } +output "cluster_name" { + description = "The name of the EKS cluster" + value = module.eks.cluster_name +} + output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster for the OpenID Connect identity provider" value = module.eks.cluster_oidc_issuer_url diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 92e85aacf6..7ae5250dce 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72" + version = ">= 4.45" } kubernetes = { source = "hashicorp/kubernetes" @@ -12,7 +12,7 @@ terraform { } helm = { source = "hashicorp/helm" - version = ">= 2.4" + version = ">= 2.7" } kubectl = { source = "gavinbunney/kubectl" diff --git a/examples/outposts/README.md b/examples/outposts/README.md new file mode 100644 index 0000000000..20b60fd4e9 --- /dev/null +++ b/examples/outposts/README.md @@ -0,0 +1,115 @@ +# EKS on Outposts + +Configuration in this directory creates an AWS EKS local cluster on AWS Outposts + +See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts.html) for further details. + +Note: This example requires an an AWS Outpost to provision. + +## Usage + +To run this example you need to: + +1. Deploy the remote host where the cluster will be provisioned from. The remote host is required since only private access is permitted to clusters created on Outposts. If you have access to the network where Outposts are provisioned (VPN, etc.), you can skip this step: + +```bash +$ cd prerequisites +$ terraform init +$ terraform plan +$ terraform apply +``` + +2. If provisioning using the remote host deployed in step 1, connect to the remote host using SSM. Note, you will need to have the [SSM plugin for the AWS CLI installed](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html). You can use the output generated by step 1 to connect: + +```bash +$ aws ssm start-session --region --target +``` + +3. Once connected to the remote host, navigate to the cloned project example directory and deploy the example: + +```bash +$ cd $HOME/terraform-aws-eks/examples/outposts +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 4.45 | +| [kubernetes](#provider\_kubernetes) | >= 2.10 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | n/a | + +## Resources + +| Name | Type | +|------|------| +| [kubernetes_storage_class_v1.this](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/storage_class_v1) | resource | +| [aws_outposts_outpost_instance_types.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/outposts_outpost_instance_types) | data source | +| [aws_outposts_outposts.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/outposts_outposts) | data source | +| [aws_subnet.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | +| [aws_subnets.lookup](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | +| [aws_subnets.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | +| [aws_vpc.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [region](#input\_region) | The AWS region to deploy into (e.g. us-east-1) | `string` | `"us-west-2"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | +| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | +| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | +| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | +| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | +| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | +| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | +| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | +| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | +| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | +| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | +| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | +| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | +| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | +| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | +| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | +| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | +| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | + diff --git a/examples/outposts/main.tf b/examples/outposts/main.tf new file mode 100644 index 0000000000..50cbc9586d --- /dev/null +++ b/examples/outposts/main.tf @@ -0,0 +1,152 @@ +provider "aws" { + region = var.region +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # Note: `cluster_id` is used with Outposts for auth + args = ["eks", "get-token", "--cluster-id", module.eks.cluster_id, "--region", var.region] + } +} + +locals { + name = "ex-${basename(path.cwd)}" + cluster_version = "1.21" # Required by EKS on Outposts + + outpost_arn = element(tolist(data.aws_outposts_outposts.this.arns), 0) + instance_type = element(tolist(data.aws_outposts_outpost_instance_types.this.instance_types), 0) + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + + cluster_endpoint_public_access = false # Not available on Outpost + cluster_endpoint_private_access = true + + vpc_id = data.aws_vpc.this.id + subnet_ids = data.aws_subnets.this.ids + + outpost_config = { + control_plane_instance_type = local.instance_type + outpost_arns = [local.outpost_arn] + } + + # Local clusters will automatically add the node group IAM role to the aws-auth configmap + manage_aws_auth_configmap = true + + # Extend cluster security group rules + cluster_security_group_additional_rules = { + ingress_vpc_https = { + description = "Remote host to control plane" + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "ingress" + cidr_blocks = [data.aws_vpc.this.cidr_block] + } + } + + self_managed_node_group_defaults = { + attach_cluster_primary_security_group = true + + iam_role_additional_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + } + + self_managed_node_groups = { + outpost = { + name = local.name + + min_size = 2 + max_size = 5 + desired_size = 3 + instance_type = local.instance_type + + # Additional information is required to join local clusters to EKS + bootstrap_extra_args = <<-EOT + --enable-local-outpost true --cluster-id ${module.eks.cluster_id} --container-runtime containerd + EOT + } + } + + tags = local.tags +} + +resource "kubernetes_storage_class_v1" "this" { + metadata { + name = "ebs-sc" + annotations = { + "storageclass.kubernetes.io/is-default-class" = "true" + } + } + + storage_provisioner = "ebs.csi.aws.com" + volume_binding_mode = "WaitForFirstConsumer" + allow_volume_expansion = true + + parameters = { + type = "gp2" + encrypted = "true" + } +} + +################################################################################ +# Supporting Resources +################################################################################ + +data "aws_outposts_outposts" "this" {} + +data "aws_outposts_outpost_instance_types" "this" { + arn = local.outpost_arn +} + +# This just grabs the first Outpost and returns its subnets +data "aws_subnets" "lookup" { + filter { + name = "outpost-arn" + values = [local.outpost_arn] + } +} + +# This grabs a single subnet to reverse lookup those that belong to same VPC +# This is whats used for the cluster +data "aws_subnet" "this" { + id = element(tolist(data.aws_subnets.lookup.ids), 0) +} + +# These are subnets for the Outpost and restricted to the same VPC +# This is whats used for the cluster +data "aws_subnets" "this" { + filter { + name = "outpost-arn" + values = [local.outpost_arn] + } + + filter { + name = "vpc-id" + values = [data.aws_subnet.this.vpc_id] + } +} + +data "aws_vpc" "this" { + id = data.aws_subnet.this.vpc_id +} diff --git a/examples/outposts/outputs.tf b/examples/outposts/outputs.tf new file mode 100644 index 0000000000..43334ecc0a --- /dev/null +++ b/examples/outposts/outputs.tf @@ -0,0 +1,211 @@ +################################################################################ +# Cluster +################################################################################ + +output "cluster_arn" { + description = "The Amazon Resource Name (ARN) of the cluster" + value = module.eks.cluster_arn +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded certificate data required to communicate with the cluster" + value = module.eks.cluster_certificate_authority_data +} + +output "cluster_endpoint" { + description = "Endpoint for your Kubernetes API server" + value = module.eks.cluster_endpoint +} + +output "cluster_id" { + description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" + value = module.eks.cluster_id +} + +output "cluster_name" { + description = "The name of the EKS cluster" + value = module.eks.cluster_name +} + +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_oidc_issuer_url +} + +output "cluster_platform_version" { + description = "Platform version for the cluster" + value = module.eks.cluster_platform_version +} + +output "cluster_status" { + description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" + value = module.eks.cluster_status +} + +output "cluster_primary_security_group_id" { + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = module.eks.cluster_primary_security_group_id +} + +################################################################################ +# KMS Key +################################################################################ + +output "kms_key_arn" { + description = "The Amazon Resource Name (ARN) of the key" + value = module.eks.kms_key_arn +} + +output "kms_key_id" { + description = "The globally unique identifier for the key" + value = module.eks.kms_key_id +} + +output "kms_key_policy" { + description = "The IAM resource policy set on the key" + value = module.eks.kms_key_policy +} + +################################################################################ +# Security Group +################################################################################ + +output "cluster_security_group_arn" { + description = "Amazon Resource Name (ARN) of the cluster security group" + value = module.eks.cluster_security_group_arn +} + +output "cluster_security_group_id" { + description = "ID of the cluster security group" + value = module.eks.cluster_security_group_id +} + +################################################################################ +# Node Security Group +################################################################################ + +output "node_security_group_arn" { + description = "Amazon Resource Name (ARN) of the node shared security group" + value = module.eks.node_security_group_arn +} + +output "node_security_group_id" { + description = "ID of the node shared security group" + value = module.eks.node_security_group_id +} + +################################################################################ +# IRSA +################################################################################ + +output "oidc_provider" { + description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" + value = module.eks.oidc_provider +} + +output "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + value = module.eks.oidc_provider_arn +} + +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = module.eks.cluster_tls_certificate_sha1_fingerprint +} + +################################################################################ +# IAM Role +################################################################################ + +output "cluster_iam_role_name" { + description = "IAM role name of the EKS cluster" + value = module.eks.cluster_iam_role_name +} + +output "cluster_iam_role_arn" { + description = "IAM role ARN of the EKS cluster" + value = module.eks.cluster_iam_role_arn +} + +output "cluster_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.cluster_iam_role_unique_id +} + +################################################################################ +# EKS Addons +################################################################################ + +output "cluster_addons" { + description = "Map of attribute maps for all EKS cluster addons enabled" + value = module.eks.cluster_addons +} + +################################################################################ +# EKS Identity Provider +################################################################################ + +output "cluster_identity_providers" { + description = "Map of attribute maps for all EKS identity providers enabled" + value = module.eks.cluster_identity_providers +} + +################################################################################ +# CloudWatch Log Group +################################################################################ + +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_name +} + +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_arn +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.eks.fargate_profiles +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +output "eks_managed_node_groups" { + description = "Map of attribute maps for all EKS managed node groups created" + value = module.eks.eks_managed_node_groups +} + +output "eks_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by EKS managed node groups" + value = module.eks.eks_managed_node_groups_autoscaling_group_names +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +output "self_managed_node_groups" { + description = "Map of attribute maps for all self managed node groups created" + value = module.eks.self_managed_node_groups +} + +output "self_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by self-managed node groups" + value = module.eks.self_managed_node_groups_autoscaling_group_names +} + +################################################################################ +# Additional +################################################################################ + +output "aws_auth_configmap_yaml" { + description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" + value = module.eks.aws_auth_configmap_yaml +} diff --git a/examples/outposts/prerequisites/main.tf b/examples/outposts/prerequisites/main.tf new file mode 100644 index 0000000000..e49d7bdbb5 --- /dev/null +++ b/examples/outposts/prerequisites/main.tf @@ -0,0 +1,150 @@ +provider "aws" { + region = var.region +} + +locals { + name = "ex-${basename(path.cwd)}" + + terraform_version = "1.3.6" + + outpost_arn = element(tolist(data.aws_outposts_outposts.this.arns), 0) + instance_type = element(tolist(data.aws_outposts_outpost_instance_types.this.instance_types), 0) + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# Pre-Requisites +################################################################################ + +module "ssm_bastion_ec2" { + source = "terraform-aws-modules/ec2-instance/aws" + version = "~> 4.2" + + name = "${local.name}-bastion" + + create_iam_instance_profile = true + iam_role_policies = { + AdministratorAccess = "arn:aws:iam::aws:policy/AdministratorAccess" + } + + instance_type = local.instance_type + + user_data = <<-EOT + #!/bin/bash + + # Add ssm-user since it won't exist until first login + adduser -m ssm-user + tee /etc/sudoers.d/ssm-agent-users <<'EOF' + # User rules for ssm-user + ssm-user ALL=(ALL) NOPASSWD:ALL + EOF + chmod 440 /etc/sudoers.d/ssm-agent-users + + cd /home/ssm-user + + # Install git to clone repo + yum install git -y + + # Install Terraform + curl -sSO https://releases.hashicorp.com/terraform/${local.terraform_version}/terraform_${local.terraform_version}_linux_amd64.zip + sudo unzip -qq terraform_${local.terraform_version}_linux_amd64.zip terraform -d /usr/bin/ + rm terraform_${local.terraform_version}_linux_amd64.zip 2> /dev/null + + # Install kubectl + curl -LO https://dl.k8s.io/release/v1.21.0/bin/linux/amd64/kubectl + install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + + # Remove default awscli which is v1 - we want latest v2 + yum remove awscli -y + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -qq awscliv2.zip + ./aws/install + + # Clone repo + git clone https://github.com/bryantbiggs/terraform-aws-eks.git \ + && cd /home/ssm-user/terraform-aws-eks \ + && git checkout refactor/v19 + + chown -R ssm-user:ssm-user /home/ssm-user/ + EOT + + vpc_security_group_ids = [module.bastion_security_group.security_group_id] + subnet_id = element(data.aws_subnets.this.ids, 0) + + tags = local.tags +} + +module "bastion_security_group" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 4.0" + + name = "${local.name}-bastion" + description = "Security group to allow provisioning ${local.name} EKS local cluster on Outposts" + vpc_id = data.aws_vpc.this.id + + ingress_with_cidr_blocks = [ + { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = data.aws_vpc.this.cidr_block + }, + ] + egress_with_cidr_blocks = [ + { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = "0.0.0.0/0" + }, + ] + + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ + +data "aws_outposts_outposts" "this" {} + +data "aws_outposts_outpost_instance_types" "this" { + arn = local.outpost_arn +} + +# This just grabs the first Outpost and returns its subnets +data "aws_subnets" "lookup" { + filter { + name = "outpost-arn" + values = [local.outpost_arn] + } +} + +# This grabs a single subnet to reverse lookup those that belong to same VPC +# This is whats used for the cluster +data "aws_subnet" "this" { + id = element(tolist(data.aws_subnets.lookup.ids), 0) +} + +# These are subnets for the Outpost and restricted to the same VPC +# This is whats used for the cluster +data "aws_subnets" "this" { + filter { + name = "outpost-arn" + values = [local.outpost_arn] + } + + filter { + name = "vpc-id" + values = [data.aws_subnet.this.vpc_id] + } +} + +data "aws_vpc" "this" { + id = data.aws_subnet.this.vpc_id +} diff --git a/examples/outposts/prerequisites/outputs.tf b/examples/outposts/prerequisites/outputs.tf new file mode 100644 index 0000000000..f2ff81ab70 --- /dev/null +++ b/examples/outposts/prerequisites/outputs.tf @@ -0,0 +1,4 @@ +output "ssm_start_session" { + description = "SSM start session command to connect to remote host created" + value = "aws ssm start-session --region ${var.region} --target ${module.ssm_bastion_ec2.id}" +} diff --git a/examples/outposts/prerequisites/variables.tf b/examples/outposts/prerequisites/variables.tf new file mode 100644 index 0000000000..47945c8501 --- /dev/null +++ b/examples/outposts/prerequisites/variables.tf @@ -0,0 +1,5 @@ +variable "region" { + description = "The AWS region to deploy into (e.g. us-east-1)" + type = string + default = "us-west-2" +} diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf new file mode 100644 index 0000000000..5f058b4c11 --- /dev/null +++ b/examples/outposts/prerequisites/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.34" + } + } +} diff --git a/examples/outposts/variables.tf b/examples/outposts/variables.tf new file mode 100644 index 0000000000..47945c8501 --- /dev/null +++ b/examples/outposts/variables.tf @@ -0,0 +1,5 @@ +variable "region" { + description = "The AWS region to deploy into (e.g. us-east-1)" + type = string + default = "us-west-2" +} diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf new file mode 100644 index 0000000000..f2f8625d4b --- /dev/null +++ b/examples/outposts/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.45" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + } +} diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 9608cdb8cd..97deaa59ab 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -25,23 +25,23 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | -| [tls](#requirement\_tls) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.72 | -| [tls](#provider\_tls) | >= 3.0 | +| [aws](#provider\_aws) | >= 4.45 | ## Modules | Name | Source | Version | |------|--------|---------| +| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 1.1 | | [eks](#module\_eks) | ../.. | n/a | +| [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources @@ -49,15 +49,11 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| | [aws_ec2_capacity_reservation.targeted](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ec2_capacity_reservation) | resource | -| [aws_key_pair.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | -| [aws_kms_key.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | -| [aws_kms_key.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | -| [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | +| [aws_iam_policy.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | ## Inputs @@ -77,9 +73,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | -| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 27e82eebb3..fe7b5d85c8 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -14,11 +14,17 @@ provider "kubernetes" { } } +data "aws_caller_identity" "current" {} +data "aws_availability_zones" "available" {} + locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" cluster_version = "1.24" region = "eu-west-1" + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + tags = { Example = local.name GithubRepo = "terraform-aws-eks" @@ -26,8 +32,6 @@ locals { } } -data "aws_caller_identity" "current" {} - ################################################################################ # EKS Module ################################################################################ @@ -35,69 +39,31 @@ data "aws_caller_identity" "current" {} module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true cluster_addons = { coredns = { - resolve_conflicts = "OVERWRITE" + most_recent = true + } + kube-proxy = { + most_recent = true } - kube-proxy = {} vpc-cni = { - resolve_conflicts = "OVERWRITE" + most_recent = true } } - cluster_encryption_config = [{ - provider_key_arn = aws_kms_key.eks.arn - resources = ["secrets"] - }] - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + control_plane_subnet_ids = module.vpc.intra_subnets # Self managed node groups will not automatically create the aws-auth configmap so we need to create_aws_auth_configmap = true manage_aws_auth_configmap = true - # Extend cluster security group rules - cluster_security_group_additional_rules = { - egress_nodes_ephemeral_ports_tcp = { - description = "To node 1025-65535" - protocol = "tcp" - from_port = 1025 - to_port = 65535 - type = "egress" - source_node_security_group = true - } - } - - # Extend node-to-node security group rules - node_security_group_additional_rules = { - ingress_self_all = { - description = "Node to node all ports/protocols" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "ingress" - self = true - } - egress_all = { - description = "Node all egress" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "egress" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] - } - } - self_managed_node_group_defaults = { - create_security_group = false - # enable discovery of autoscaling groups by cluster-autoscaler autoscaling_group_tags = { "k8s.io/cluster-autoscaler/enabled" : true, @@ -117,24 +83,31 @@ module "eks" { ami_id = data.aws_ami.eks_default_bottlerocket.id instance_type = "m5.large" desired_size = 2 - key_name = aws_key_pair.this.key_name - - iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"] + key_name = module.key_pair.key_pair_name bootstrap_extra_args = <<-EOT - # The admin host container provides SSH access and runs with "superpowers". - # It is disabled by default, but can be disabled explicitly. - [settings.host-containers.admin] - enabled = false - - # The control host container provides out-of-band access via SSM. - # It is enabled by default, and can be disabled if you do not expect to use SSM. - # This could leave you with no way to access the API and change settings on an existing node! - [settings.host-containers.control] - enabled = true - - [settings.kubernetes.node-labels] - ingress = "allowed" + # The admin host container provides SSH access and runs with "superpowers". + # It is disabled by default, but can be disabled explicitly. + [settings.host-containers.admin] + enabled = false + + # The control host container provides out-of-band access via SSM. + # It is enabled by default, and can be disabled if you do not expect to use SSM. + # This could leave you with no way to access the API and change settings on an existing node! + [settings.host-containers.control] + enabled = true + + # extra args added + [settings.kernel] + lockdown = "integrity" + + [settings.kubernetes.node-labels] + label1 = "foo" + label2 = "bar" + + [settings.kubernetes.node-taints] + dedicated = "experimental:PreferNoSchedule" + special = "true:NoSchedule" EOT } @@ -177,15 +150,14 @@ module "eks" { instance_type = "c5n.9xlarge" post_bootstrap_user_data = <<-EOT - - # Install EFA - curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz - tar -xf aws-efa-installer-latest.tar.gz && cd aws-efa-installer - ./efa_installer.sh -y --minimal - fi_info -p efa -t FI_EP_RDM - - # Disable ptrace - sysctl -w kernel.yama.ptrace_scope=0 + # Install EFA + curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz + tar -xf aws-efa-installer-latest.tar.gz && cd aws-efa-installer + ./efa_installer.sh -y --minimal + fi_info -p efa -t FI_EP_RDM + + # Disable ptrace + sysctl -w kernel.yama.ptrace_scope=0 EOT network_interfaces = [ @@ -214,12 +186,12 @@ module "eks" { bootstrap_extra_args = "--kubelet-extra-args '--max-pods=110'" pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false EOT post_bootstrap_user_data = <<-EOT - echo "you are free little kubelet!" + echo "you are free little kubelet!" EOT instance_type = "m6i.large" @@ -228,9 +200,8 @@ module "eks" { launch_template_use_name_prefix = true launch_template_description = "Self managed node group example launch template" - ebs_optimized = true - vpc_security_group_ids = [aws_security_group.additional.id] - enable_monitoring = true + ebs_optimized = true + enable_monitoring = true block_device_mappings = { xvda = { @@ -241,7 +212,7 @@ module "eks" { iops = 3000 throughput = 150 encrypted = true - kms_key_id = aws_kms_key.ebs.arn + kms_key_id = module.ebs_kms_key.key_id delete_on_termination = true } } @@ -267,34 +238,9 @@ module "eks" { iam_role_tags = { Purpose = "Protector of the kubelet" } - iam_role_additional_policies = [ - "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" - ] - - create_security_group = true - security_group_name = "self-managed-node-group-complete-example" - security_group_use_name_prefix = false - security_group_description = "Self managed node group complete example security group" - security_group_rules = { - phoneOut = { - description = "Hello CloudFlare" - protocol = "udp" - from_port = 53 - to_port = 53 - type = "egress" - cidr_blocks = ["1.1.1.1/32"] - } - phoneHome = { - description = "Hello cluster" - protocol = "udp" - from_port = 53 - to_port = 53 - type = "egress" - source_cluster_security_group = true # bit of reflection lookup - } - } - security_group_tags = { - Purpose = "Protector of the kubelet" + iam_role_additional_policies = { + AmazonEC2ContainerRegistryReadOnly = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + additional = aws_iam_policy.additional.arn } timeouts = { @@ -321,11 +267,12 @@ module "vpc" { version = "~> 3.0" name = local.name - cidr = "10.0.0.0/16" + cidr = local.vpc_cidr - azs = ["${local.region}a", "${local.region}b", "${local.region}c"] - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] enable_nat_gateway = true single_nat_gateway = true @@ -336,44 +283,16 @@ module "vpc" { create_flow_log_cloudwatch_log_group = true public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = 1 + "kubernetes.io/role/elb" = 1 } private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = 1 - } - - tags = local.tags -} - -resource "aws_security_group" "additional" { - name_prefix = "${local.name}-additional" - vpc_id = module.vpc.vpc_id - - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = [ - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - ] + "kubernetes.io/role/internal-elb" = 1 } tags = local.tags } -resource "aws_kms_key" "eks" { - description = "EKS Secret Encryption Key" - deletion_window_in_days = 7 - enable_key_rotation = true - - tags = local.tags -} - data "aws_ami" "eks_default" { most_recent = true owners = ["amazon"] @@ -394,19 +313,37 @@ data "aws_ami" "eks_default_bottlerocket" { } } -resource "tls_private_key" "this" { - algorithm = "RSA" -} +module "key_pair" { + source = "terraform-aws-modules/key-pair/aws" + version = "~> 2.0" -resource "aws_key_pair" "this" { - key_name = local.name - public_key = tls_private_key.this.public_key_openssh + key_name_prefix = local.name + create_private_key = true + + tags = local.tags } -resource "aws_kms_key" "ebs" { - description = "Customer managed key to encrypt self managed node group volumes" - deletion_window_in_days = 7 - policy = data.aws_iam_policy_document.ebs.json +module "ebs_kms_key" { + source = "terraform-aws-modules/kms/aws" + version = "~> 1.1" + + description = "Customer managed key to encrypt EKS managed node group volumes" + + # Policy + key_administrators = [ + data.aws_caller_identity.current.arn + ] + key_service_users = [ + # required for the ASG to manage encrypted volumes for nodes + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", + # required for the cluster / persistentvolume-controller to create encrypted PVCs + module.eks.cluster_iam_role_arn, + ] + + # Aliases + aliases = ["eks/${local.name}/ebs"] + + tags = local.tags } resource "aws_ec2_capacity_reservation" "targeted" { @@ -417,58 +354,22 @@ resource "aws_ec2_capacity_reservation" "targeted" { instance_match_criteria = "targeted" } -# This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes -data "aws_iam_policy_document" "ebs" { - # Copy of default KMS policy that lets you manage it - statement { - sid = "Enable IAM User Permissions" - actions = ["kms:*"] - resources = ["*"] - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] - } - } +resource "aws_iam_policy" "additional" { + name = "${local.name}-additional" + description = "Example usage of node additional policy" - # Required for EKS - statement { - sid = "Allow service-linked role use of the CMK" - actions = [ - "kms:Encrypt", - "kms:Decrypt", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:DescribeKey" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:Describe*", + ] + Effect = "Allow" + Resource = "*" + }, ] - resources = ["*"] - - principals { - type = "AWS" - identifiers = [ - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes - module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs - ] - } - } + }) - statement { - sid = "Allow attachment of persistent resources" - actions = ["kms:CreateGrant"] - resources = ["*"] - - principals { - type = "AWS" - identifiers = [ - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes - module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs - ] - } - - condition { - test = "Bool" - variable = "kms:GrantIsForAWSResource" - values = ["true"] - } - } + tags = local.tags } diff --git a/examples/self_managed_node_group/outputs.tf b/examples/self_managed_node_group/outputs.tf index 67de532779..43334ecc0a 100644 --- a/examples/self_managed_node_group/outputs.tf +++ b/examples/self_managed_node_group/outputs.tf @@ -17,16 +17,16 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } -output "cluster_name" { - description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" - value = module.eks.cluster_name -} - output "cluster_id" { - description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" value = module.eks.cluster_id } +output "cluster_name" { + description = "The name of the EKS cluster" + value = module.eks.cluster_name +} + output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster for the OpenID Connect identity provider" value = module.eks.cluster_oidc_issuer_url diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index fde7af0f23..f2f8625d4b 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -1,14 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72" - } - tls = { - source = "hashicorp/tls" - version = ">= 3.0" + version = ">= 4.45" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/user_data/README.md b/examples/user_data/README.md index 54cd9ec72b..cea7dce755 100644 --- a/examples/user_data/README.md +++ b/examples/user_data/README.md @@ -17,8 +17,7 @@ $ terraform apply | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | +| [terraform](#requirement\_terraform) | >= 1.0 | ## Providers diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index 5c5b1266cf..cb565ba04c 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -19,7 +19,7 @@ module "eks_mng_linux_additional" { source = "../../modules/_user_data" pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" + export CONTAINER_RUNTIME="containerd" EOT } @@ -34,14 +34,14 @@ module "eks_mng_linux_custom_ami" { enable_bootstrap_user_data = true pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false EOT bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20 --instance-type t3a.large'" post_bootstrap_user_data = <<-EOT - echo "All done" + echo "All done" EOT } @@ -56,14 +56,14 @@ module "eks_mng_linux_custom_template" { user_data_template_path = "${path.module}/templates/linux_custom.tpl" pre_bootstrap_user_data = <<-EOT - echo "foo" - export FOO=bar + echo "foo" + export FOO=bar EOT bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" post_bootstrap_user_data = <<-EOT - echo "All done" + echo "All done" EOT } @@ -80,9 +80,9 @@ module "eks_mng_bottlerocket_additional" { platform = "bottlerocket" bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" + # extra args added + [settings.kernel] + lockdown = "integrity" EOT } @@ -98,9 +98,9 @@ module "eks_mng_bottlerocket_custom_ami" { enable_bootstrap_user_data = true bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" + # extra args added + [settings.kernel] + lockdown = "integrity" EOT } @@ -116,9 +116,9 @@ module "eks_mng_bottlerocket_custom_template" { user_data_template_path = "${path.module}/templates/bottlerocket_custom.tpl" bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" + # extra args added + [settings.kernel] + lockdown = "integrity" EOT } @@ -140,14 +140,14 @@ module "self_mng_linux_bootstrap" { cluster_auth_base64 = local.cluster_auth_base64 pre_bootstrap_user_data = <<-EOT - echo "foo" - export FOO=bar + echo "foo" + export FOO=bar EOT bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" post_bootstrap_user_data = <<-EOT - echo "All done" + echo "All done" EOT } @@ -164,14 +164,14 @@ module "self_mng_linux_custom_template" { user_data_template_path = "${path.module}/templates/linux_custom.tpl" pre_bootstrap_user_data = <<-EOT - echo "foo" - export FOO=bar + echo "foo" + export FOO=bar EOT bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" post_bootstrap_user_data = <<-EOT - echo "All done" + echo "All done" EOT } @@ -197,9 +197,9 @@ module "self_mng_bottlerocket_bootstrap" { cluster_auth_base64 = local.cluster_auth_base64 bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" + # extra args added + [settings.kernel] + lockdown = "integrity" EOT } @@ -218,9 +218,9 @@ module "self_mng_bottlerocket_custom_template" { user_data_template_path = "${path.module}/templates/bottlerocket_custom.tpl" bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" + # extra args added + [settings.kernel] + lockdown = "integrity" EOT } @@ -246,13 +246,13 @@ module "self_mng_windows_bootstrap" { cluster_auth_base64 = local.cluster_auth_base64 pre_bootstrap_user_data = <<-EOT - [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' EOT # I don't know if this is the right way on WindowsOS, but its just a string check here anyways bootstrap_extra_args = "-KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot" post_bootstrap_user_data = <<-EOT - [string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + [string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' EOT } @@ -271,12 +271,12 @@ module "self_mng_windows_custom_template" { user_data_template_path = "${path.module}/templates/windows_custom.tpl" pre_bootstrap_user_data = <<-EOT - [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' EOT # I don't know if this is the right way on WindowsOS, but its just a string check here anyways bootstrap_extra_args = "-KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot" post_bootstrap_user_data = <<-EOT - [string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + [string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' EOT } diff --git a/examples/user_data/versions.tf b/examples/user_data/versions.tf index 22e8d7265f..7117131f4c 100644 --- a/examples/user_data/versions.tf +++ b/examples/user_data/versions.tf @@ -1,10 +1,3 @@ terraform { - required_version = ">= 0.13.1" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.72" - } - } + required_version = ">= 1.0" } diff --git a/main.tf b/main.tf index 3ed4edaacf..9193031348 100644 --- a/main.tf +++ b/main.tf @@ -1,11 +1,13 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} -data "aws_default_tags" "current" {} locals { create = var.create && var.putin_khuylo cluster_role = try(aws_iam_role.this[0].arn, var.iam_role_arn) + + create_outposts_local_cluster = length(var.outpost_config) > 0 + enable_cluster_encryption_config = length(var.cluster_encryption_config) > 0 && !local.create_outposts_local_cluster } ################################################################################ @@ -25,16 +27,32 @@ resource "aws_eks_cluster" "this" { subnet_ids = coalescelist(var.control_plane_subnet_ids, var.subnet_ids) endpoint_private_access = var.cluster_endpoint_private_access endpoint_public_access = var.cluster_endpoint_public_access - public_access_cidrs = var.cluster_endpoint_public_access_cidrs + public_access_cidrs = var.cluster_endpoint_public_access ? var.cluster_endpoint_public_access_cidrs : [] } - kubernetes_network_config { - ip_family = var.cluster_ip_family - service_ipv4_cidr = var.cluster_service_ipv4_cidr + dynamic "kubernetes_network_config" { + # Not valid on Outposts + for_each = local.create_outposts_local_cluster ? [] : [1] + + content { + ip_family = var.cluster_ip_family + service_ipv4_cidr = var.cluster_service_ipv4_cidr + service_ipv6_cidr = var.cluster_service_ipv6_cidr + } + } + + dynamic "outpost_config" { + for_each = local.create_outposts_local_cluster ? [var.outpost_config] : [] + + content { + control_plane_instance_type = outpost_config.value.control_plane_instance_type + outpost_arns = outpost_config.value.outpost_arns + } } dynamic "encryption_config" { - for_each = toset(var.cluster_encryption_config) + # Not available on Outposts + for_each = local.enable_cluster_encryption_config ? [var.cluster_encryption_config] : [] content { provider { @@ -67,9 +85,8 @@ resource "aws_ec2_tag" "cluster_primary_security_group" { # This should not affect the name of the cluster primary security group # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2006 # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2008 - # `aws_default_tags` is merged in to "dedupe" tags and stabilize tag updates - for_each = { for k, v in merge(var.tags, var.cluster_tags, data.aws_default_tags.current.tags) : - k => v if local.create && k != "Name" && var.create_cluster_primary_security_group_tags + for_each = { for k, v in merge(var.tags, var.cluster_tags) : + k => v if local.create && k != "Name" && var.create_cluster_primary_security_group_tags && v != null } resource_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id @@ -93,9 +110,9 @@ resource "aws_cloudwatch_log_group" "this" { module "kms" { source = "terraform-aws-modules/kms/aws" - version = "1.0.2" # Note - be mindful of Terraform/provider version compatibility between modules + version = "1.1.0" # Note - be mindful of Terraform/provider version compatibility between modules - create = local.create && var.create_kms_key + create = local.create && var.create_kms_key && !local.create_outposts_local_cluster # not valid on Outposts description = coalesce(var.kms_key_description, "${var.cluster_name} cluster encryption key") key_usage = "ENCRYPT_DECRYPT" @@ -129,7 +146,7 @@ locals { cluster_security_group_id = local.create_cluster_sg ? aws_security_group.cluster[0].id : var.cluster_security_group_id # Do not add rules to node security group if the module is not creating it - cluster_security_group_rules = local.create_node_sg ? { + cluster_security_group_rules = { for k, v in { ingress_nodes_443 = { description = "Node groups to cluster API" protocol = "tcp" @@ -138,23 +155,7 @@ locals { type = "ingress" source_node_security_group = true } - egress_nodes_443 = { - description = "Cluster API to node groups" - protocol = "tcp" - from_port = 443 - to_port = 443 - type = "egress" - source_node_security_group = true - } - egress_nodes_kubelet = { - description = "Cluster API to node kubelets" - protocol = "tcp" - from_port = 10250 - to_port = 10250 - type = "egress" - source_node_security_group = true - } - } : {} + } : k => v if local.create_node_sg } } resource "aws_security_group" "cluster" { @@ -177,7 +178,10 @@ resource "aws_security_group" "cluster" { } resource "aws_security_group_rule" "cluster" { - for_each = { for k, v in merge(local.cluster_security_group_rules, var.cluster_security_group_additional_rules) : k => v if local.create_cluster_sg } + for_each = { for k, v in merge( + local.cluster_security_group_rules, + var.cluster_security_group_additional_rules + ) : k => v if local.create_cluster_sg } # Required security_group_id = aws_security_group.cluster[0].id @@ -187,15 +191,13 @@ resource "aws_security_group_rule" "cluster" { type = each.value.type # Optional - description = try(each.value.description, null) - cidr_blocks = try(each.value.cidr_blocks, null) - ipv6_cidr_blocks = try(each.value.ipv6_cidr_blocks, null) - prefix_list_ids = try(each.value.prefix_list_ids, []) - self = try(each.value.self, null) - source_security_group_id = try( - each.value.source_security_group_id, - try(each.value.source_node_security_group, false) ? local.node_security_group_id : null - ) + description = lookup(each.value, "description", null) + cidr_blocks = lookup(each.value, "cidr_blocks", null) + ipv6_cidr_blocks = lookup(each.value, "ipv6_cidr_blocks", null) + prefix_list_ids = lookup(each.value, "prefix_list_ids", []) + self = lookup(each.value, "self", null) + source_security_group_id = lookup(each.value, "source_security_group_id", + lookup(each.value, "source_node_security_group", false)) ? local.node_security_group_id : null } ################################################################################ @@ -204,13 +206,15 @@ resource "aws_security_group_rule" "cluster" { ################################################################################ data "tls_certificate" "this" { - count = local.create && var.enable_irsa ? 1 : 0 + # Not available on outposts + count = local.create && var.enable_irsa && !local.create_outposts_local_cluster ? 1 : 0 url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer } resource "aws_iam_openid_connect_provider" "oidc_provider" { - count = local.create && var.enable_irsa ? 1 : 0 + # Not available on outposts + count = local.create && var.enable_irsa && !local.create_outposts_local_cluster ? 1 : 0 client_id_list = distinct(compact(concat(["sts.${local.dns_suffix}"], var.openid_connect_audiences))) thumbprint_list = concat(data.tls_certificate.this[0].certificates[*].sha1_fingerprint, var.custom_oidc_thumbprints) @@ -227,9 +231,9 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { ################################################################################ locals { - create_iam_role = local.create && var.create_iam_role - iam_role_name = coalesce(var.iam_role_name, "${var.cluster_name}-cluster") - policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + create_iam_role = local.create && var.create_iam_role + iam_role_name = coalesce(var.iam_role_name, "${var.cluster_name}-cluster") + iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" cluster_encryption_policy_name = coalesce(var.cluster_encryption_policy_name, "${local.iam_role_name}-ClusterEncryption") @@ -249,6 +253,17 @@ data "aws_iam_policy_document" "assume_role_policy" { type = "Service" identifiers = ["eks.${local.dns_suffix}"] } + + dynamic "principals" { + for_each = local.create_outposts_local_cluster ? [1] : [] + + content { + type = "Service" + identifiers = [ + "ec2.${local.dns_suffix}", + ] + } + } } } @@ -292,10 +307,17 @@ resource "aws_iam_role" "this" { # Policies attached ref https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html resource "aws_iam_role_policy_attachment" "this" { - for_each = local.create_iam_role ? toset(compact(distinct(concat([ - "${local.policy_arn_prefix}/AmazonEKSClusterPolicy", - "${local.policy_arn_prefix}/AmazonEKSVPCResourceController", - ], var.iam_role_additional_policies)))) : toset([]) + for_each = { for k, v in { + AmazonEKSClusterPolicy = local.create_outposts_local_cluster ? "${local.iam_role_policy_prefix}/AmazonEKSLocalOutpostClusterPolicy" : "${local.iam_role_policy_prefix}/AmazonEKSClusterPolicy", + AmazonEKSVPCResourceController = "${local.iam_role_policy_prefix}/AmazonEKSVPCResourceController", + } : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_role } policy_arn = each.value role = aws_iam_role.this[0].name @@ -303,14 +325,16 @@ resource "aws_iam_role_policy_attachment" "this" { # Using separate attachment due to `The "for_each" value depends on resource attributes that cannot be determined until apply` resource "aws_iam_role_policy_attachment" "cluster_encryption" { - count = local.create_iam_role && var.attach_cluster_encryption_policy && length(var.cluster_encryption_config) > 0 ? 1 : 0 + # Encryption config not available on Outposts + count = local.create_iam_role && var.attach_cluster_encryption_policy && local.enable_cluster_encryption_config ? 1 : 0 policy_arn = aws_iam_policy.cluster_encryption[0].arn role = aws_iam_role.this[0].name } resource "aws_iam_policy" "cluster_encryption" { - count = local.create_iam_role && var.attach_cluster_encryption_policy && length(var.cluster_encryption_config) > 0 ? 1 : 0 + # Encryption config not available on Outposts + count = local.create_iam_role && var.attach_cluster_encryption_policy && local.enable_cluster_encryption_config ? 1 : 0 name = var.cluster_encryption_policy_use_name_prefix ? null : local.cluster_encryption_policy_name name_prefix = var.cluster_encryption_policy_use_name_prefix ? local.cluster_encryption_policy_name : null @@ -328,7 +352,7 @@ resource "aws_iam_policy" "cluster_encryption" { "kms:DescribeKey", ] Effect = "Allow" - Resource = var.create_kms_key ? [module.kms.key_arn] : [for config in var.cluster_encryption_config : config.provider_key_arn] + Resource = var.create_kms_key ? module.kms.key_arn : var.cluster_encryption_config.provider_key_arn }, ] }) @@ -341,14 +365,22 @@ resource "aws_iam_policy" "cluster_encryption" { ################################################################################ resource "aws_eks_addon" "this" { - for_each = { for k, v in var.cluster_addons : k => v if local.create } + # Not supported on outposts + for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } cluster_name = aws_eks_cluster.this[0].name addon_name = try(each.value.name, each.key) - addon_version = lookup(each.value, "addon_version", null) - resolve_conflicts = lookup(each.value, "resolve_conflicts", null) - service_account_role_arn = lookup(each.value, "service_account_role_arn", null) + addon_version = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) + preserve = try(each.value.preserve, null) + resolve_conflicts = try(each.value.resolve_conflicts, "OVERWRITE") + service_account_role_arn = try(each.value.service_account_role_arn, null) + + timeouts { + create = try(each.value.timeouts.create, var.cluster_addons_timeouts.create, null) + update = try(each.value.timeouts.update, var.cluster_addons_timeouts.update, null) + delete = try(each.value.timeouts.delete, var.cluster_addons_timeouts.delete, null) + } depends_on = [ module.fargate_profile, @@ -359,13 +391,21 @@ resource "aws_eks_addon" "this" { tags = var.tags } +data "aws_eks_addon_version" "this" { + for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } + + addon_name = try(each.value.name, each.key) + kubernetes_version = coalesce(var.cluster_version, aws_eks_cluster.this[0].version) + most_recent = try(each.value.most_recent, null) +} + ################################################################################ # EKS Identity Provider # Note - this is different from IRSA ################################################################################ resource "aws_eks_identity_provider_config" "this" { - for_each = { for k, v in var.cluster_identity_providers : k => v if local.create } + for_each = { for k, v in var.cluster_identity_providers : k => v if local.create && !local.create_outposts_local_cluster } cluster_name = aws_eks_cluster.this[0].name diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 87da77b447..0853fd9e1a 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -9,7 +9,7 @@ See [`examples/user_data/`](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [terraform](#requirement\_terraform) | >= 1.0 | | [cloudinit](#requirement\_cloudinit) | >= 2.0 | ## Providers diff --git a/modules/_user_data/outputs.tf b/modules/_user_data/outputs.tf index c2a569b05b..075801b233 100644 --- a/modules/_user_data/outputs.tf +++ b/modules/_user_data/outputs.tf @@ -1,4 +1,4 @@ output "user_data" { description = "Base64 encoded user data rendered for the provided inputs" - value = try(local.platform[var.platform].user_data, "") + value = try(local.platform[var.platform].user_data, null) } diff --git a/modules/_user_data/versions.tf b/modules/_user_data/versions.tf index e293dc67ce..2dbd12cdc0 100644 --- a/modules/_user_data/versions.tf +++ b/modules/_user_data/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { cloudinit = { diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index bf8d6de605..d4d4cc9b40 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -18,7 +18,18 @@ module "eks_managed_node_group" { // The following variables are necessary if you decide to use the module outside of the parent EKS module context. // Without it, the security groups of the nodes are empty and thus won't join the cluster. cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id - cluster_security_group_id = module.eks.node_security_group_id + cluster_security_group_id = module.eks.node_security_group_id + + // Note: `disk_size`, and `remote_access` can only be set when using the EKS managed node group default launch template + // This module defaults to providing a custom launch template to allow for custom security groups, tag propagation, etc. + // use_custom_launch_template = false + // disk_size = 50 + // + // # Remote access cannot be specified with a launch template + // remote_access = { + // ec2_ssh_key = module.key_pair.key_pair_name + // source_security_group_ids = [aws_security_group.remote_access.id] + // } min_size = 1 max_size = 10 @@ -53,14 +64,14 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.72 | +| [aws](#provider\_aws) | >= 4.45 | ## Modules @@ -74,10 +85,9 @@ module "eks_managed_node_group" { |------|------| | [aws_eks_node_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | -| [aws_security_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_security_group_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -98,26 +108,24 @@ module "eks_managed_node_group" { | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | | [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | -| [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes version. Defaults to EKS Cluster Kubernetes version | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | | [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | -| [create\_security\_group](#input\_create\_security\_group) | Determines whether to create a security group | `bool` | `true` | no | | [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | | [desired\_size](#input\_desired\_size) | Desired number of instances/nodes | `number` | `1` | no | | [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | -| [disk\_size](#input\_disk\_size) | Disk size in GiB for nodes. Defaults to `20` | `number` | `null` | no | +| [disk\_size](#input\_disk\_size) | Disk size in GiB for nodes. Defaults to `20`. Only valid when `use_custom_launch_template` = `false` | `number` | `null` | no | | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance(s) will be EBS-optimized | `bool` | `null` | no | -| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `map(string)` | `{}` | no | +| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | | [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | | [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template. Only valid when using a custom AMI via `ami_id` | `bool` | `false` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | | [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | | [force\_update\_version](#input\_force\_update\_version) | Force version update if existing pods are unable to be drained due to a pod disruption budget issue | `bool` | `null` | no | -| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the node group. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | @@ -133,11 +141,13 @@ module "eks_managed_node_group" { | [labels](#input\_labels) | Key-value map of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed | `map(string)` | `null` | no | | [launch\_template\_default\_version](#input\_launch\_template\_default\_version) | Default version of the launch template | `string` | `null` | no | | [launch\_template\_description](#input\_launch\_template\_description) | Description of the launch template | `string` | `null` | no | -| [launch\_template\_name](#input\_launch\_template\_name) | Launch template name - either to be created (`var.create_launch_template` = `true`) or existing (`var.create_launch_template` = `false`) | `string` | `""` | no | +| [launch\_template\_id](#input\_launch\_template\_id) | The ID of an existing launch template to use. Required when `create_launch_template` = `false` and `use_custom_launch_template` = `true` | `string` | `""` | no | +| [launch\_template\_name](#input\_launch\_template\_name) | Name of launch template to be created | `string` | `null` | no | | [launch\_template\_tags](#input\_launch\_template\_tags) | A map of additional tags to add to the tag\_specifications of launch template created | `map(string)` | `{}` | no | | [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | | [launch\_template\_version](#input\_launch\_template\_version) | Launch template version number. The default is `$Default` | `string` | `null` | no | -| [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with | `map(string)` | `{}` | no | +| [license\_specifications](#input\_license\_specifications) | A map of license specifications to associate with | `any` | `{}` | no | +| [maintenance\_options](#input\_maintenance\_options) | The maintenance options for the instance | `any` | `{}` | no | | [max\_size](#input\_max\_size) | Maximum number of instances/nodes | `number` | `3` | no | | [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | | [min\_size](#input\_min\_size) | Minimum number of instances/nodes | `number` | `0` | no | @@ -147,22 +157,18 @@ module "eks_managed_node_group" { | [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket` or `linux` based; `windows` is not supported | `string` | `"linux"` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | | [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet | `map(string)` | `{}` | no | | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | -| [remote\_access](#input\_remote\_access) | Configuration block with remote access settings | `any` | `{}` | no | -| [security\_group\_description](#input\_security\_group\_description) | Description for the security group created | `string` | `"EKS managed node group security group"` | no | -| [security\_group\_name](#input\_security\_group\_name) | Name to use on security group created | `string` | `null` | no | -| [security\_group\_rules](#input\_security\_group\_rules) | List of security group rules to add to the security group created | `any` | `{}` | no | -| [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the security group created | `map(string)` | `{}` | no | -| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `bool` | `true` | no | +| [remote\_access](#input\_remote\_access) | Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false` | `any` | `{}` | no | | [subnet\_ids](#input\_subnet\_ids) | Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME` | `list(string)` | `null` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [taints](#input\_taints) | The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group | `any` | `{}` | no | | [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the node group | `map(string)` | `{}` | no | -| [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates | `map(string)` | `{}` | no | +| [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates | `map(string)` |
{
"max_unavailable_percentage": 33
}
| no | | [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update the launch templates default version on each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | +| [use\_custom\_launch\_template](#input\_use\_custom\_launch\_template) | Determines whether to use a custom launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | | [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix | `bool` | `true` | no | | [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | -| [vpc\_id](#input\_vpc\_id) | ID of the VPC where the security group/nodes will be provisioned | `string` | `null` | no | | [vpc\_security\_group\_ids](#input\_vpc\_security\_group\_ids) | A list of security group IDs to associate | `list(string)` | `[]` | no | ## Outputs @@ -183,6 +189,4 @@ module "eks_managed_node_group" { | [node\_group\_resources](#output\_node\_group\_resources) | List of objects containing information about underlying resources | | [node\_group\_status](#output\_node\_group\_status) | Status of the EKS Node Group | | [node\_group\_taints](#output\_node\_group\_taints) | List of objects containing information about taints applied to the node group | -| [security\_group\_arn](#output\_security\_group\_arn) | Amazon Resource Name (ARN) of the security group | -| [security\_group\_id](#output\_security\_group\_id) | ID of the security group | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index f55630876e..06275f00a2 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -1,5 +1,4 @@ data "aws_partition" "current" {} - data "aws_caller_identity" "current" {} ################################################################################ @@ -30,75 +29,51 @@ module "user_data" { ################################################################################ locals { - # There are 4 scenarios here that have to be considered for `use_custom_launch_template`: - # 1. `var.create_launch_template = false && var.launch_template_name == ""` => EKS MNG will use its own default LT - # 2. `var.create_launch_template = false && var.launch_template_name == "something"` => User provided custom LT will be used - # 3. `var.create_launch_template = true && var.launch_template_name == ""` => Custom LT will be used, module will provide a default name - # 4. `var.create_launch_template = true && var.launch_template_name == "something"` => Custom LT will be used, LT name is provided by user - use_custom_launch_template = var.create_launch_template || var.launch_template_name != "" - - launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-eks-node-group") - - security_group_ids = compact(concat([try(aws_security_group.this[0].id, ""), var.cluster_primary_security_group_id], var.vpc_security_group_ids)) + launch_template_name = coalesce(var.launch_template_name, "${var.name}-eks-node-group") + security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) } resource "aws_launch_template" "this" { - count = var.create && var.create_launch_template ? 1 : 0 - - name = var.launch_template_use_name_prefix ? null : local.launch_template_name_int - name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name_int}-" : null - description = var.launch_template_description - - ebs_optimized = var.ebs_optimized - image_id = var.ami_id - # # Set on node group instead - # instance_type = var.launch_template_instance_type - key_name = var.key_name - user_data = module.user_data.user_data - - vpc_security_group_ids = length(var.network_interfaces) > 0 ? [] : local.security_group_ids - - default_version = var.launch_template_default_version - update_default_version = var.update_launch_template_default_version - disable_api_termination = var.disable_api_termination - # Set on EKS managed node group, will fail if set here - # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics - # instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior - kernel_id = var.kernel_id - ram_disk_id = var.ram_disk_id + count = var.create && var.create_launch_template && var.use_custom_launch_template ? 1 : 0 dynamic "block_device_mappings" { for_each = var.block_device_mappings + content { - device_name = block_device_mappings.value.device_name - no_device = lookup(block_device_mappings.value, "no_device", null) - virtual_name = lookup(block_device_mappings.value, "virtual_name", null) + device_name = try(block_device_mappings.value.device_name, null) dynamic "ebs" { - for_each = flatten([lookup(block_device_mappings.value, "ebs", [])]) + for_each = try([block_device_mappings.value.ebs], []) + content { - delete_on_termination = lookup(ebs.value, "delete_on_termination", null) - encrypted = lookup(ebs.value, "encrypted", null) - kms_key_id = lookup(ebs.value, "kms_key_id", null) - iops = lookup(ebs.value, "iops", null) - throughput = lookup(ebs.value, "throughput", null) - snapshot_id = lookup(ebs.value, "snapshot_id", null) - volume_size = lookup(ebs.value, "volume_size", null) - volume_type = lookup(ebs.value, "volume_type", null) + delete_on_termination = try(ebs.value.delete_on_termination, null) + encrypted = try(ebs.value.encrypted, null) + iops = try(ebs.value.iops, null) + kms_key_id = try(ebs.value.kms_key_id, null) + snapshot_id = try(ebs.value.snapshot_id, null) + throughput = try(ebs.value.throughput, null) + volume_size = try(ebs.value.volume_size, null) + volume_type = try(ebs.value.volume_type, null) } } + + no_device = try(block_device_mappings.value.no_device, null) + virtual_name = try(block_device_mappings.value.virtual_name, null) } } dynamic "capacity_reservation_specification" { for_each = length(var.capacity_reservation_specification) > 0 ? [var.capacity_reservation_specification] : [] + content { - capacity_reservation_preference = lookup(capacity_reservation_specification.value, "capacity_reservation_preference", null) + capacity_reservation_preference = try(capacity_reservation_specification.value.capacity_reservation_preference, null) dynamic "capacity_reservation_target" { for_each = try([capacity_reservation_specification.value.capacity_reservation_target], []) + content { - capacity_reservation_id = lookup(capacity_reservation_target.value, "capacity_reservation_id", null) + capacity_reservation_id = try(capacity_reservation_target.value.capacity_reservation_id, null) + capacity_reservation_resource_group_arn = try(capacity_reservation_target.value.capacity_reservation_resource_group_arn, null) } } } @@ -106,21 +81,29 @@ resource "aws_launch_template" "this" { dynamic "cpu_options" { for_each = length(var.cpu_options) > 0 ? [var.cpu_options] : [] + content { - core_count = cpu_options.value.core_count - threads_per_core = cpu_options.value.threads_per_core + core_count = try(cpu_options.value.core_count, null) + threads_per_core = try(cpu_options.value.threads_per_core, null) } } dynamic "credit_specification" { for_each = length(var.credit_specification) > 0 ? [var.credit_specification] : [] + content { - cpu_credits = credit_specification.value.cpu_credits + cpu_credits = try(credit_specification.value.cpu_credits, null) } } + default_version = var.launch_template_default_version + description = var.launch_template_description + disable_api_termination = var.disable_api_termination + ebs_optimized = var.ebs_optimized + dynamic "elastic_gpu_specifications" { - for_each = length(var.elastic_gpu_specifications) > 0 ? [var.elastic_gpu_specifications] : [] + for_each = var.elastic_gpu_specifications + content { type = elastic_gpu_specifications.value.type } @@ -128,6 +111,7 @@ resource "aws_launch_template" "this" { dynamic "elastic_inference_accelerator" { for_each = length(var.elastic_inference_accelerator) > 0 ? [var.elastic_inference_accelerator] : [] + content { type = elastic_inference_accelerator.value.type } @@ -135,6 +119,7 @@ resource "aws_launch_template" "this" { dynamic "enclave_options" { for_each = length(var.enclave_options) > 0 ? [var.enclave_options] : [] + content { enabled = enclave_options.value.enabled } @@ -143,7 +128,8 @@ resource "aws_launch_template" "this" { # Set on EKS managed node group, will fail if set here # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics # dynamic "hibernation_options" { - # for_each = var.hibernation_options != null ? [var.hibernation_options] : [] + # for_each = length(var.hibernation_options) > 0 ? [var.hibernation_options] : [] + # content { # configured = hibernation_options.value.configured # } @@ -159,104 +145,154 @@ resource "aws_launch_template" "this" { # } # } + image_id = var.ami_id + # Set on EKS managed node group, will fail if set here + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics + # instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior + dynamic "instance_market_options" { for_each = length(var.instance_market_options) > 0 ? [var.instance_market_options] : [] + content { - market_type = instance_market_options.value.market_type + market_type = try(instance_market_options.value.market_type, null) dynamic "spot_options" { - for_each = length(lookup(instance_market_options.value, "spot_options", {})) > 0 ? [instance_market_options.value.spot_options] : [] + for_each = try([instance_market_options.value.spot_options], []) + content { - block_duration_minutes = lookup(spot_options.value, "block_duration_minutes", null) - instance_interruption_behavior = lookup(spot_options.value, "instance_interruption_behavior", null) - max_price = lookup(spot_options.value, "max_price", null) - spot_instance_type = lookup(spot_options.value, "spot_instance_type", null) - valid_until = lookup(spot_options.value, "valid_until", null) + block_duration_minutes = try(spot_options.value.block_duration_minutes, null) + instance_interruption_behavior = try(spot_options.value.instance_interruption_behavior, null) + max_price = try(spot_options.value.max_price, null) + spot_instance_type = try(spot_options.value.spot_instance_type, null) + valid_until = try(spot_options.value.valid_until, null) } } } } + # # Set on node group instead + # instance_type = var.launch_template_instance_type + kernel_id = var.kernel_id + key_name = var.key_name + dynamic "license_specification" { - for_each = length(var.license_specifications) > 0 ? [var.license_specifications] : [] + for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} + content { license_configuration_arn = license_specifications.value.license_configuration_arn } } + dynamic "maintenance_options" { + for_each = length(var.maintenance_options) > 0 ? [var.maintenance_options] : [] + + content { + auto_recovery = try(maintenance_options.value.auto_recovery, null) + } + } + dynamic "metadata_options" { for_each = length(var.metadata_options) > 0 ? [var.metadata_options] : [] + content { - http_endpoint = lookup(metadata_options.value, "http_endpoint", null) - http_tokens = lookup(metadata_options.value, "http_tokens", null) - http_put_response_hop_limit = lookup(metadata_options.value, "http_put_response_hop_limit", null) - http_protocol_ipv6 = lookup(metadata_options.value, "http_protocol_ipv6", null) - instance_metadata_tags = lookup(metadata_options.value, "instance_metadata_tags", null) + http_endpoint = try(metadata_options.value.http_endpoint, null) + http_protocol_ipv6 = try(metadata_options.value.http_protocol_ipv6, null) + http_put_response_hop_limit = try(metadata_options.value.http_put_response_hop_limit, null) + http_tokens = try(metadata_options.value.http_tokens, null) + instance_metadata_tags = try(metadata_options.value.instance_metadata_tags, null) } } dynamic "monitoring" { - for_each = var.enable_monitoring != null ? [1] : [] + for_each = var.enable_monitoring ? [1] : [] + content { enabled = var.enable_monitoring } } + name = var.launch_template_use_name_prefix ? null : local.launch_template_name + name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null + dynamic "network_interfaces" { for_each = var.network_interfaces content { - associate_carrier_ip_address = lookup(network_interfaces.value, "associate_carrier_ip_address", null) - associate_public_ip_address = lookup(network_interfaces.value, "associate_public_ip_address", null) - delete_on_termination = lookup(network_interfaces.value, "delete_on_termination", null) - description = lookup(network_interfaces.value, "description", null) - device_index = lookup(network_interfaces.value, "device_index", null) - interface_type = lookup(network_interfaces.value, "interface_type", null) + associate_carrier_ip_address = try(network_interfaces.value.associate_carrier_ip_address, null) + associate_public_ip_address = try(network_interfaces.value.associate_public_ip_address, null) + delete_on_termination = try(network_interfaces.value.delete_on_termination, null) + description = try(network_interfaces.value.description, null) + device_index = try(network_interfaces.value.device_index, null) + interface_type = try(network_interfaces.value.interface_type, null) + ipv4_address_count = try(network_interfaces.value.ipv4_address_count, null) ipv4_addresses = try(network_interfaces.value.ipv4_addresses, []) - ipv4_address_count = lookup(network_interfaces.value, "ipv4_address_count", null) + ipv4_prefix_count = try(network_interfaces.value.ipv4_prefix_count, null) + ipv4_prefixes = try(network_interfaces.value.ipv4_prefixes, null) + ipv6_address_count = try(network_interfaces.value.ipv6_address_count, null) ipv6_addresses = try(network_interfaces.value.ipv6_addresses, []) - ipv6_address_count = lookup(network_interfaces.value, "ipv6_address_count", null) - network_interface_id = lookup(network_interfaces.value, "network_interface_id", null) - private_ip_address = lookup(network_interfaces.value, "private_ip_address", null) - security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) + ipv6_prefix_count = try(network_interfaces.value.ipv6_prefix_count, null) + ipv6_prefixes = try(network_interfaces.value.ipv6_prefixes, []) + network_card_index = try(network_interfaces.value.network_card_index, null) + network_interface_id = try(network_interfaces.value.network_interface_id, null) + private_ip_address = try(network_interfaces.value.private_ip_address, null) + # Ref: https://github.com/hashicorp/terraform-provider-aws/issues/4570 + security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) # Set on EKS managed node group, will fail if set here # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics - # subnet_id = lookup(network_interfaces.value, "subnet_id", null) + # subnet_id = try(network_interfaces.value.subnet_id, null) } } dynamic "placement" { for_each = length(var.placement) > 0 ? [var.placement] : [] + + content { + affinity = try(placement.value.affinity, null) + availability_zone = try(placement.value.availability_zone, null) + group_name = try(placement.value.group_name, null) + host_id = try(placement.value.host_id, null) + host_resource_group_arn = try(placement.value.host_resource_group_arn, null) + partition_number = try(placement.value.partition_number, null) + spread_domain = try(placement.value.spread_domain, null) + tenancy = try(placement.value.tenancy, null) + } + } + + dynamic "private_dns_name_options" { + for_each = length(var.private_dns_name_options) > 0 ? [var.private_dns_name_options] : [] + content { - affinity = lookup(placement.value, "affinity", null) - availability_zone = lookup(placement.value, "availability_zone", null) - group_name = lookup(placement.value, "group_name", null) - host_id = lookup(placement.value, "host_id", null) - spread_domain = lookup(placement.value, "spread_domain", null) - tenancy = lookup(placement.value, "tenancy", null) - partition_number = lookup(placement.value, "partition_number", null) + enable_resource_name_dns_aaaa_record = try(private_dns_name_options.value.enable_resource_name_dns_aaaa_record, null) + enable_resource_name_dns_a_record = try(private_dns_name_options.value.enable_resource_name_dns_a_record, null) + hostname_type = try(private_dns_name_options.value.hostname_type, null) } } + ram_disk_id = var.ram_disk_id + dynamic "tag_specifications" { for_each = toset(["instance", "volume", "network-interface"]) + content { resource_type = tag_specifications.key tags = merge(var.tags, { Name = var.name }, var.launch_template_tags) } } - lifecycle { - create_before_destroy = true - } + update_default_version = var.update_launch_template_default_version + user_data = module.user_data.user_data + vpc_security_group_ids = length(var.network_interfaces) > 0 ? [] : local.security_group_ids + + tags = var.tags - # Prevent premature access of security group roles and policies by pods that + # Prevent premature access of policies by pods that # require permissions on create/destroy that depend on nodes depends_on = [ - aws_security_group_rule.this, aws_iam_role_policy_attachment.this, ] - tags = var.tags + lifecycle { + create_before_destroy = true + } } ################################################################################ @@ -264,7 +300,7 @@ resource "aws_launch_template" "this" { ################################################################################ locals { - launch_template_name = try(aws_launch_template.this[0].name, var.launch_template_name, null) + launch_template_id = var.create && var.create_launch_template ? aws_launch_template.this[0].id : var.launch_template_id # Change order to allow users to set version priority before using defaults launch_template_version = coalesce(var.launch_template_version, try(aws_launch_template.this[0].default_version, "$Default")) } @@ -293,21 +329,23 @@ resource "aws_eks_node_group" "this" { version = var.ami_id != "" ? null : var.cluster_version capacity_type = var.capacity_type - disk_size = local.use_custom_launch_template ? null : var.disk_size # if using LT, set disk size on LT or else it will error here + disk_size = var.use_custom_launch_template ? null : var.disk_size # if using a custom LT, set disk size on custom LT or else it will error here force_update_version = var.force_update_version instance_types = var.instance_types labels = var.labels dynamic "launch_template" { - for_each = local.use_custom_launch_template ? [1] : [] + for_each = var.use_custom_launch_template ? [1] : [] + content { - name = local.launch_template_name + id = local.launch_template_id version = local.launch_template_version } } dynamic "remote_access" { for_each = length(var.remote_access) > 0 ? [var.remote_access] : [] + content { ec2_ssh_key = try(remote_access.value.ec2_ssh_key, null) source_security_group_ids = try(remote_access.value.source_security_group_ids, []) @@ -316,15 +354,17 @@ resource "aws_eks_node_group" "this" { dynamic "taint" { for_each = var.taints + content { key = taint.value.key - value = lookup(taint.value, "value") + value = try(taint.value.value, null) effect = taint.value.effect } } dynamic "update_config" { for_each = length(var.update_config) > 0 ? [var.update_config] : [] + content { max_unavailable_percentage = try(update_config.value.max_unavailable_percentage, null) max_unavailable = try(update_config.value.max_unavailable, null) @@ -350,68 +390,14 @@ resource "aws_eks_node_group" "this" { ) } -################################################################################ -# Security Group -################################################################################ - -locals { - security_group_name = coalesce(var.security_group_name, "${var.name}-eks-node-group") - create_security_group = var.create && var.create_security_group -} - -resource "aws_security_group" "this" { - count = local.create_security_group ? 1 : 0 - - name = var.security_group_use_name_prefix ? null : local.security_group_name - name_prefix = var.security_group_use_name_prefix ? "${local.security_group_name}-" : null - description = var.security_group_description - vpc_id = var.vpc_id - - tags = merge( - var.tags, - { "Name" = local.security_group_name }, - var.security_group_tags - ) - - # https://github.com/hashicorp/terraform-provider-aws/issues/2445 - # https://github.com/hashicorp/terraform-provider-aws/issues/9692 - lifecycle { - create_before_destroy = true - } -} - -resource "aws_security_group_rule" "this" { - for_each = { for k, v in var.security_group_rules : k => v if local.create_security_group } - - # Required - security_group_id = aws_security_group.this[0].id - protocol = each.value.protocol - from_port = each.value.from_port - to_port = each.value.to_port - type = each.value.type - - # Optional - description = try(each.value.description, null) - cidr_blocks = try(each.value.cidr_blocks, null) - ipv6_cidr_blocks = try(each.value.ipv6_cidr_blocks, null) - prefix_list_ids = try(each.value.prefix_list_ids, []) - self = try(each.value.self, null) - source_security_group_id = try( - each.value.source_security_group_id, - try(each.value.source_cluster_security_group, false) ? var.cluster_security_group_id : null - ) -} - ################################################################################ # IAM Role ################################################################################ locals { - iam_role_name = coalesce(var.iam_role_name, "${var.name}-eks-node-group") - + iam_role_name = coalesce(var.iam_role_name, "${var.name}-eks-node-group") iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" - - cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } data "aws_iam_policy_document" "assume_role_policy" { @@ -445,11 +431,18 @@ resource "aws_iam_role" "this" { # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group resource "aws_iam_role_policy_attachment" "this" { - for_each = var.create && var.create_iam_role ? toset(compact(distinct(concat([ + for_each = { for k, v in toset(compact([ "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", var.iam_role_attach_cni_policy ? local.cni_policy : "", - ], var.iam_role_additional_policies)))) : toset([]) + ])) : k => v if var.create && var.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_additional_policies : k => v if var.create && var.create_iam_role } policy_arn = each.value role = aws_iam_role.this[0].name diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index 9d7535332f..58dbe9646f 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -4,22 +4,22 @@ output "launch_template_id" { description = "The ID of the launch template" - value = try(aws_launch_template.this[0].id, "") + value = try(aws_launch_template.this[0].id, null) } output "launch_template_arn" { description = "The ARN of the launch template" - value = try(aws_launch_template.this[0].arn, "") + value = try(aws_launch_template.this[0].arn, null) } output "launch_template_latest_version" { description = "The latest version of the launch template" - value = try(aws_launch_template.this[0].latest_version, "") + value = try(aws_launch_template.this[0].latest_version, null) } output "launch_template_name" { description = "The name of the launch template" - value = try(aws_launch_template.this[0].name, "") + value = try(aws_launch_template.this[0].name, null) } ################################################################################ @@ -28,17 +28,17 @@ output "launch_template_name" { output "node_group_arn" { description = "Amazon Resource Name (ARN) of the EKS Node Group" - value = try(aws_eks_node_group.this[0].arn, "") + value = try(aws_eks_node_group.this[0].arn, null) } output "node_group_id" { description = "EKS Cluster name and EKS Node Group name separated by a colon (`:`)" - value = try(aws_eks_node_group.this[0].id, "") + value = try(aws_eks_node_group.this[0].id, null) } output "node_group_resources" { description = "List of objects containing information about underlying resources" - value = try(aws_eks_node_group.this[0].resources, "") + value = try(aws_eks_node_group.this[0].resources, null) } output "node_group_autoscaling_group_names" { @@ -48,7 +48,7 @@ output "node_group_autoscaling_group_names" { output "node_group_status" { description = "Status of the EKS Node Group" - value = try(aws_eks_node_group.this[0].arn, "") + value = try(aws_eks_node_group.this[0].arn, null) } output "node_group_labels" { @@ -61,27 +61,13 @@ output "node_group_taints" { value = try(aws_eks_node_group.this[0].taint, []) } -################################################################################ -# Security Group -################################################################################ - -output "security_group_arn" { - description = "Amazon Resource Name (ARN) of the security group" - value = try(aws_security_group.this[0].arn, "") -} - -output "security_group_id" { - description = "ID of the security group" - value = try(aws_security_group.this[0].id, "") -} - ################################################################################ # IAM Role ################################################################################ output "iam_role_name" { description = "The name of the IAM role" - value = try(aws_iam_role.this[0].name, "") + value = try(aws_iam_role.this[0].name, null) } output "iam_role_arn" { @@ -91,5 +77,5 @@ output "iam_role_arn" { output "iam_role_unique_id" { description = "Stable and unique string identifying the IAM role" - value = try(aws_iam_role.this[0].unique_id, "") + value = try(aws_iam_role.this[0].unique_id, null) } diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 0000827719..d2790e1acb 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -84,12 +84,24 @@ variable "create_launch_template" { default = true } -variable "launch_template_name" { - description = "Launch template name - either to be created (`var.create_launch_template` = `true`) or existing (`var.create_launch_template` = `false`)" +variable "use_custom_launch_template" { + description = "Determines whether to use a custom launch template or not. If set to `false`, EKS will use its own default launch template" + type = bool + default = true +} + +variable "launch_template_id" { + description = "The ID of an existing launch template to use. Required when `create_launch_template` = `false` and `use_custom_launch_template` = `true`" type = string default = "" } +variable "launch_template_name" { + description = "Name of launch template to be created" + type = string + default = null +} + variable "launch_template_use_name_prefix" { description = "Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix" type = bool @@ -188,7 +200,7 @@ variable "credit_specification" { variable "elastic_gpu_specifications" { description = "The elastic GPU to attach to the instance" - type = map(string) + type = any default = {} } @@ -210,9 +222,15 @@ variable "instance_market_options" { default = {} } +variable "maintenance_options" { + description = "The maintenance options for the instance" + type = any + default = {} +} + variable "license_specifications" { - description = "A list of license specifications to associate with" - type = map(string) + description = "A map of license specifications to associate with" + type = any default = {} } @@ -244,6 +262,12 @@ variable "placement" { default = {} } +variable "private_dns_name_options" { + description = "The options for the instance hostname. The default values are inherited from the subnet" + type = map(string) + default = {} +} + variable "launch_template_tags" { description = "A map of additional tags to add to the tag_specifications of launch template created" type = map(string) @@ -309,7 +333,7 @@ variable "capacity_type" { } variable "disk_size" { - description = "Disk size in GiB for nodes. Defaults to `20`" + description = "Disk size in GiB for nodes. Defaults to `20`. Only valid when `use_custom_launch_template` = `false`" type = number default = null } @@ -345,7 +369,7 @@ variable "launch_template_version" { } variable "remote_access" { - description = "Configuration block with remote access settings" + description = "Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false`" type = any default = {} } @@ -359,7 +383,9 @@ variable "taints" { variable "update_config" { description = "Configuration block of settings for max unavailable resources during node group updates" type = map(string) - default = {} + default = { + max_unavailable_percentage = 33 + } } variable "timeouts" { @@ -368,58 +394,6 @@ variable "timeouts" { default = {} } -################################################################################ -# Security Group -################################################################################ - -variable "create_security_group" { - description = "Determines whether to create a security group" - type = bool - default = true -} - -variable "security_group_name" { - description = "Name to use on security group created" - type = string - default = null -} - -variable "security_group_use_name_prefix" { - description = "Determines whether the security group name (`security_group_name`) is used as a prefix" - type = bool - default = true -} - -variable "security_group_description" { - description = "Description for the security group created" - type = string - default = "EKS managed node group security group" -} - -variable "vpc_id" { - description = "ID of the VPC where the security group/nodes will be provisioned" - type = string - default = null -} - -variable "security_group_rules" { - description = "List of security group rules to add to the security group created" - type = any - default = {} -} - -variable "cluster_security_group_id" { - description = "Cluster control plane security group ID" - type = string - default = null -} - -variable "security_group_tags" { - description = "A map of additional tags to add to the security group created" - type = map(string) - default = {} -} - ################################################################################ # IAM Role ################################################################################ @@ -480,8 +454,8 @@ variable "iam_role_attach_cni_policy" { variable "iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" - type = list(string) - default = [] + type = map(string) + default = {} } variable "iam_role_tags" { diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 22e8d7265f..325eee94e1 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72" + version = ">= 4.45" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 97dd0d3e53..4ed9a6fd2d 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -8,8 +8,8 @@ Configuration in this directory creates a Fargate EKS Profile module "fargate_profile" { source = "terraform-aws-modules/eks/aws//modules/fargate-profile" - name = "separate-fargate-profile" - cluster_name = "my-cluster" + name = "separate-fargate-profile" + cluster_name = "my-cluster" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] selectors = [{ @@ -28,14 +28,14 @@ module "fargate_profile" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.72 | +| [aws](#provider\_aws) | >= 4.45 | ## Modules @@ -47,6 +47,7 @@ No modules. |------|------| | [aws_eks_fargate_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_fargate_profile) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | @@ -60,7 +61,7 @@ No modules. | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `null` | no | | [create](#input\_create) | Determines whether to create Fargate profile or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | -| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the Fargate profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | diff --git a/modules/fargate-profile/main.tf b/modules/fargate-profile/main.tf index 554b0e8814..de9dd2d754 100644 --- a/modules/fargate-profile/main.tf +++ b/modules/fargate-profile/main.tf @@ -1,13 +1,10 @@ data "aws_partition" "current" {} - data "aws_caller_identity" "current" {} locals { - iam_role_name = coalesce(var.iam_role_name, var.name, "fargate-profile") - + iam_role_name = coalesce(var.iam_role_name, var.name, "fargate-profile") iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" - - cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } ################################################################################ @@ -44,10 +41,17 @@ resource "aws_iam_role" "this" { } resource "aws_iam_role_policy_attachment" "this" { - for_each = var.create && var.create_iam_role ? toset(compact(distinct(concat([ + for_each = { for k, v in toset(compact([ "${local.iam_role_policy_prefix}/AmazonEKSFargatePodExecutionRolePolicy", var.iam_role_attach_cni_policy ? local.cni_policy : "", - ], var.iam_role_additional_policies)))) : toset([]) + ])) : k => v if var.create && var.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_additional_policies : k => v if var.create && var.create_iam_role } policy_arn = each.value role = aws_iam_role.this[0].name diff --git a/modules/fargate-profile/outputs.tf b/modules/fargate-profile/outputs.tf index c8b663e80e..96763bfb1f 100644 --- a/modules/fargate-profile/outputs.tf +++ b/modules/fargate-profile/outputs.tf @@ -4,7 +4,7 @@ output "iam_role_name" { description = "The name of the IAM role" - value = try(aws_iam_role.this[0].name, "") + value = try(aws_iam_role.this[0].name, null) } output "iam_role_arn" { @@ -14,7 +14,7 @@ output "iam_role_arn" { output "iam_role_unique_id" { description = "Stable and unique string identifying the IAM role" - value = try(aws_iam_role.this[0].unique_id, "") + value = try(aws_iam_role.this[0].unique_id, null) } ################################################################################ @@ -23,20 +23,20 @@ output "iam_role_unique_id" { output "fargate_profile_arn" { description = "Amazon Resource Name (ARN) of the EKS Fargate Profile" - value = try(aws_eks_fargate_profile.this[0].arn, "") + value = try(aws_eks_fargate_profile.this[0].arn, null) } output "fargate_profile_id" { description = "EKS Cluster name and EKS Fargate Profile name separated by a colon (`:`)" - value = try(aws_eks_fargate_profile.this[0].id, "") + value = try(aws_eks_fargate_profile.this[0].id, null) } output "fargate_profile_status" { description = "Status of the EKS Fargate Profile" - value = try(aws_eks_fargate_profile.this[0].status, "") + value = try(aws_eks_fargate_profile.this[0].status, null) } output "fargate_profile_pod_execution_role_arn" { description = "Amazon Resource Name (ARN) of the EKS Fargate Profile Pod execution role ARN" - value = try(aws_eks_fargate_profile.this[0].pod_execution_role_arn, "") + value = try(aws_eks_fargate_profile.this[0].pod_execution_role_arn, null) } diff --git a/modules/fargate-profile/variables.tf b/modules/fargate-profile/variables.tf index 4d5e95ccbe..e22279dc6b 100644 --- a/modules/fargate-profile/variables.tf +++ b/modules/fargate-profile/variables.tf @@ -70,8 +70,8 @@ variable "iam_role_attach_cni_policy" { variable "iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" - type = list(string) - default = [] + type = map(string) + default = {} } variable "iam_role_tags" { diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 22e8d7265f..325eee94e1 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72" + version = ">= 4.45" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 1797c6d2e8..71f4ff9455 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -99,14 +99,14 @@ module "karpenter" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.72 | +| [aws](#provider\_aws) | >= 4.45 | ## Modules diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index d54fa9585a..62640f0550 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -261,11 +261,14 @@ locals { resource "aws_cloudwatch_event_rule" "this" { for_each = { for k, v in local.events : k => v if local.enable_spot_termination } - name = "Karpenter${each.value.name}-${var.cluster_name}" + name_prefix = "Karpenter${each.value.name}-" description = each.value.description event_pattern = jsonencode(each.value.event_pattern) - tags = var.tags + tags = merge( + { "ClusterName" : var.cluster_name }, + var.tags, + ) } resource "aws_cloudwatch_event_target" "this" { diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 22e8d7265f..325eee94e1 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72" + version = ">= 4.45" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 4c392362f6..051aa4f4eb 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -16,8 +16,11 @@ module "self_managed_node_group" { vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + // The following variables are necessary if you decide to use the module outside of the parent EKS module context. + // Without it, the security groups of the nodes are empty and thus won't join the cluster. vpc_security_group_ids = [ - # cluster_security_group_id, + module.eks.cluster_primary_security_group_id, + module.eks.cluster_security_group_id, ] min_size = 1 @@ -39,14 +42,14 @@ module "self_managed_node_group" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 3.72 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.45 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.72 | +| [aws](#provider\_aws) | >= 4.45 | ## Modules @@ -62,13 +65,11 @@ module "self_managed_node_group" { | [aws_autoscaling_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_schedule) | resource | | [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | -| [aws_security_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_security_group_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_default_tags.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -88,32 +89,33 @@ module "self_managed_node_group" { | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `""` | no | | [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | -| [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Cluster control plane security group ID | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | +| [context](#input\_context) | Reserved | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | | [create](#input\_create) | Determines whether to create self managed node group or not | `bool` | `true` | no | | [create\_autoscaling\_group](#input\_create\_autoscaling\_group) | Determines whether to create autoscaling group or not | `bool` | `true` | no | | [create\_iam\_instance\_profile](#input\_create\_iam\_instance\_profile) | Determines whether an IAM instance profile is created or to use an existing IAM instance profile | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create launch template or not | `bool` | `true` | no | | [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | -| [create\_security\_group](#input\_create\_security\_group) | Determines whether to create a security group | `bool` | `true` | no | | [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | | [default\_cooldown](#input\_default\_cooldown) | The amount of time, in seconds, after a scaling activity completes before another scaling activity can start | `number` | `null` | no | +| [default\_instance\_warmup](#input\_default\_instance\_warmup) | Amount of time, in seconds, until a newly launched instance can contribute to the Amazon CloudWatch metrics. This delay lets an instance finish initializing before Amazon EC2 Auto Scaling aggregates instance metrics, resulting in more reliable usage data | `number` | `null` | no | | [delete\_timeout](#input\_delete\_timeout) | Delete timeout to wait for destroying autoscaling group | `string` | `null` | no | | [desired\_size](#input\_desired\_size) | The number of Amazon EC2 instances that should be running in the autoscaling group | `number` | `1` | no | | [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance will be EBS-optimized | `bool` | `null` | no | -| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `map(string)` | `{}` | no | +| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | | [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | | [enabled\_metrics](#input\_enabled\_metrics) | A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances` | `list(string)` | `[]` | no | | [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | | [force\_delete](#input\_force\_delete) | Allows deleting the Auto Scaling Group without waiting for all instances in the pool to terminate. You can force an Auto Scaling Group to delete even if it's in the process of scaling a resource. Normally, Terraform drains all the instances before deleting the group. This bypasses that behavior and potentially leaves resources dangling | `bool` | `null` | no | +| [force\_delete\_warm\_pool](#input\_force\_delete\_warm\_pool) | Allows deleting the Auto Scaling Group without waiting for all instances in the warm pool to terminate | `bool` | `null` | no | | [health\_check\_grace\_period](#input\_health\_check\_grace\_period) | Time (in seconds) after instance comes into service before checking health | `number` | `null` | no | | [health\_check\_type](#input\_health\_check\_type) | `EC2` or `ELB`. Controls how health checking is done | `string` | `null` | no | | [hibernation\_options](#input\_hibernation\_options) | The hibernation options for the instance | `map(string)` | `{}` | no | | [iam\_instance\_profile\_arn](#input\_iam\_instance\_profile\_arn) | Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false` | `string` | `null` | no | -| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | @@ -124,17 +126,20 @@ module "self_managed_node_group" { | [initial\_lifecycle\_hooks](#input\_initial\_lifecycle\_hooks) | One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource | `list(map(string))` | `[]` | no | | [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | | [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `{}` | no | -| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated | `any` | `{}` | no | +| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated | `any` |
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | +| [instance\_requirements](#input\_instance\_requirements) | The attribute requirements for the type of instance. If present then `instance_type` cannot be present | `any` | `{}` | no | | [instance\_type](#input\_instance\_type) | The type of the instance to launch | `string` | `""` | no | | [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | | [key\_name](#input\_key\_name) | The key name that should be used for the instance | `string` | `null` | no | | [launch\_template\_default\_version](#input\_launch\_template\_default\_version) | Default Version of the launch template | `string` | `null` | no | | [launch\_template\_description](#input\_launch\_template\_description) | Description of the launch template | `string` | `null` | no | -| [launch\_template\_name](#input\_launch\_template\_name) | Launch template name - either to be created (`var.create_launch_template` = `true`) or existing (`var.create_launch_template` = `false`) | `string` | `null` | no | +| [launch\_template\_id](#input\_launch\_template\_id) | The ID of an existing launch template to use. Required when `create_launch_template` = `false` | `string` | `""` | no | +| [launch\_template\_name](#input\_launch\_template\_name) | Name of launch template to be created | `string` | `null` | no | | [launch\_template\_tags](#input\_launch\_template\_tags) | A map of additional tags to add to the tag\_specifications of launch template created | `map(string)` | `{}` | no | | [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | | [launch\_template\_version](#input\_launch\_template\_version) | Launch template version. Can be version number, `$Latest`, or `$Default` | `string` | `null` | no | -| [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with | `map(string)` | `{}` | no | +| [license\_specifications](#input\_license\_specifications) | A map of license specifications to associate with | `any` | `{}` | no | +| [maintenance\_options](#input\_maintenance\_options) | The maintenance options for the instance | `any` | `{}` | no | | [max\_instance\_lifetime](#input\_max\_instance\_lifetime) | The maximum amount of time, in seconds, that an instance can be in service, values must be either equal to 0 or between 604800 and 31536000 seconds | `number` | `null` | no | | [max\_size](#input\_max\_size) | The maximum size of the autoscaling group | `number` | `3` | no | | [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | @@ -149,14 +154,10 @@ module "self_managed_node_group" { | [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | `string` | `"linux"` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | | [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet | `map(string)` | `{}` | no | | [protect\_from\_scale\_in](#input\_protect\_from\_scale\_in) | Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events. | `bool` | `false` | no | | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | | [schedules](#input\_schedules) | Map of autoscaling group schedule to create | `map(any)` | `{}` | no | -| [security\_group\_description](#input\_security\_group\_description) | Description for the security group created | `string` | `"EKS self-managed node group security group"` | no | -| [security\_group\_name](#input\_security\_group\_name) | Name to use on security group created | `string` | `null` | no | -| [security\_group\_rules](#input\_security\_group\_rules) | List of security group rules to add to the security group created | `any` | `{}` | no | -| [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the security group created | `map(string)` | `{}` | no | -| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `bool` | `true` | no | | [service\_linked\_role\_arn](#input\_service\_linked\_role\_arn) | The ARN of the service-linked role that the ASG will use to call other AWS services | `string` | `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones` | `list(string)` | `null` | no | | [suspended\_processes](#input\_suspended\_processes) | A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly | `list(string)` | `[]` | no | @@ -164,11 +165,9 @@ module "self_managed_node_group" { | [target\_group\_arns](#input\_target\_group\_arns) | A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing | `list(string)` | `[]` | no | | [termination\_policies](#input\_termination\_policies) | A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default` | `list(string)` | `[]` | no | | [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update Default Version each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | -| [use\_default\_tags](#input\_use\_default\_tags) | Enables/disables the use of provider default tags in the tag\_specifications of the Auto Scaling group | `bool` | `false` | no | | [use\_mixed\_instances\_policy](#input\_use\_mixed\_instances\_policy) | Determines whether to use a mixed instances policy in the autoscaling group or not | `bool` | `false` | no | | [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix | `bool` | `true` | no | | [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | -| [vpc\_id](#input\_vpc\_id) | ID of the VPC where the security group/nodes will be provisioned | `string` | `null` | no | | [vpc\_security\_group\_ids](#input\_vpc\_security\_group\_ids) | A list of security group IDs to associate | `list(string)` | `[]` | no | | [wait\_for\_capacity\_timeout](#input\_wait\_for\_capacity\_timeout) | A maximum duration that Terraform should wait for ASG instances to be healthy before timing out. (See also Waiting for Capacity below.) Setting this to '0' causes Terraform to skip all Capacity Waiting behavior. | `string` | `null` | no | | [wait\_for\_elb\_capacity](#input\_wait\_for\_elb\_capacity) | Setting this will cause Terraform to wait for exactly this number of healthy instances in all attached load balancers on both create and update operations. Takes precedence over `min_elb_capacity` behavior. | `number` | `null` | no | @@ -202,7 +201,5 @@ module "self_managed_node_group" { | [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | | [launch\_template\_name](#output\_launch\_template\_name) | The name of the launch template | | [platform](#output\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | -| [security\_group\_arn](#output\_security\_group\_arn) | Amazon Resource Name (ARN) of the security group | -| [security\_group\_id](#output\_security\_group\_id) | ID of the security group | | [user\_data](#output\_user\_data) | Base64 encoded user data | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index f74fb8189e..7632d24609 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -1,9 +1,6 @@ data "aws_partition" "current" {} - data "aws_caller_identity" "current" {} -data "aws_default_tags" "current" {} - data "aws_ami" "eks_default" { count = var.create ? 1 : 0 @@ -43,65 +40,51 @@ module "user_data" { ################################################################################ locals { - launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-node-group") - - security_group_ids = compact(concat([try(aws_security_group.this[0].id, ""), var.cluster_primary_security_group_id], var.vpc_security_group_ids)) + launch_template_name = coalesce(var.launch_template_name, "${var.name}-node-group") + security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) } resource "aws_launch_template" "this" { count = var.create && var.create_launch_template ? 1 : 0 - name = var.launch_template_use_name_prefix ? null : local.launch_template_name_int - name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name_int}-" : null - description = var.launch_template_description - - ebs_optimized = var.ebs_optimized - image_id = coalesce(var.ami_id, data.aws_ami.eks_default[0].image_id) - instance_type = var.instance_type - key_name = var.key_name - user_data = module.user_data.user_data - - vpc_security_group_ids = length(var.network_interfaces) > 0 ? [] : local.security_group_ids - - default_version = var.launch_template_default_version - update_default_version = var.update_launch_template_default_version - disable_api_termination = var.disable_api_termination - instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior - kernel_id = var.kernel_id - ram_disk_id = var.ram_disk_id - dynamic "block_device_mappings" { for_each = var.block_device_mappings + content { - device_name = block_device_mappings.value.device_name - no_device = lookup(block_device_mappings.value, "no_device", null) - virtual_name = lookup(block_device_mappings.value, "virtual_name", null) + device_name = try(block_device_mappings.value.device_name, null) dynamic "ebs" { - for_each = flatten([lookup(block_device_mappings.value, "ebs", [])]) + for_each = try([block_device_mappings.value.ebs], []) + content { - delete_on_termination = lookup(ebs.value, "delete_on_termination", null) - encrypted = lookup(ebs.value, "encrypted", null) - kms_key_id = lookup(ebs.value, "kms_key_id", null) - iops = lookup(ebs.value, "iops", null) - throughput = lookup(ebs.value, "throughput", null) - snapshot_id = lookup(ebs.value, "snapshot_id", null) - volume_size = lookup(ebs.value, "volume_size", null) - volume_type = lookup(ebs.value, "volume_type", null) + delete_on_termination = try(ebs.value.delete_on_termination, null) + encrypted = try(ebs.value.encrypted, null) + iops = try(ebs.value.iops, null) + kms_key_id = try(ebs.value.kms_key_id, null) + snapshot_id = try(ebs.value.snapshot_id, null) + throughput = try(ebs.value.throughput, null) + volume_size = try(ebs.value.volume_size, null) + volume_type = try(ebs.value.volume_type, null) } } + + no_device = try(block_device_mappings.value.no_device, null) + virtual_name = try(block_device_mappings.value.virtual_name, null) } } dynamic "capacity_reservation_specification" { for_each = length(var.capacity_reservation_specification) > 0 ? [var.capacity_reservation_specification] : [] + content { - capacity_reservation_preference = lookup(capacity_reservation_specification.value, "capacity_reservation_preference", null) + capacity_reservation_preference = try(capacity_reservation_specification.value.capacity_reservation_preference, null) dynamic "capacity_reservation_target" { for_each = try([capacity_reservation_specification.value.capacity_reservation_target], []) + content { - capacity_reservation_id = lookup(capacity_reservation_target.value, "capacity_reservation_id", null) + capacity_reservation_id = try(capacity_reservation_target.value.capacity_reservation_id, null) + capacity_reservation_resource_group_arn = try(capacity_reservation_target.value.capacity_reservation_resource_group_arn, null) } } } @@ -109,21 +92,29 @@ resource "aws_launch_template" "this" { dynamic "cpu_options" { for_each = length(var.cpu_options) > 0 ? [var.cpu_options] : [] + content { - core_count = cpu_options.value.core_count - threads_per_core = cpu_options.value.threads_per_core + core_count = try(cpu_options.value.core_count, null) + threads_per_core = try(cpu_options.value.threads_per_core, null) } } dynamic "credit_specification" { for_each = length(var.credit_specification) > 0 ? [var.credit_specification] : [] + content { - cpu_credits = credit_specification.value.cpu_credits + cpu_credits = try(credit_specification.value.cpu_credits, null) } } + default_version = var.launch_template_default_version + description = var.launch_template_description + disable_api_termination = var.disable_api_termination + ebs_optimized = var.ebs_optimized + dynamic "elastic_gpu_specifications" { - for_each = length(var.elastic_gpu_specifications) > 0 ? [var.elastic_gpu_specifications] : [] + for_each = var.elastic_gpu_specifications + content { type = elastic_gpu_specifications.value.type } @@ -131,6 +122,7 @@ resource "aws_launch_template" "this" { dynamic "elastic_inference_accelerator" { for_each = length(var.elastic_inference_accelerator) > 0 ? [var.elastic_inference_accelerator] : [] + content { type = elastic_inference_accelerator.value.type } @@ -138,6 +130,7 @@ resource "aws_launch_template" "this" { dynamic "enclave_options" { for_each = length(var.enclave_options) > 0 ? [var.enclave_options] : [] + content { enabled = enclave_options.value.enabled } @@ -145,6 +138,7 @@ resource "aws_launch_template" "this" { dynamic "hibernation_options" { for_each = length(var.hibernation_options) > 0 ? [var.hibernation_options] : [] + content { configured = hibernation_options.value.configured } @@ -154,102 +148,245 @@ resource "aws_launch_template" "this" { arn = var.create_iam_instance_profile ? aws_iam_instance_profile.this[0].arn : var.iam_instance_profile_arn } + image_id = coalesce(var.ami_id, data.aws_ami.eks_default[0].image_id) + instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior + dynamic "instance_market_options" { for_each = length(var.instance_market_options) > 0 ? [var.instance_market_options] : [] + content { - market_type = instance_market_options.value.market_type + market_type = try(instance_market_options.value.market_type, null) dynamic "spot_options" { - for_each = length(lookup(instance_market_options.value, "spot_options", {})) > 0 ? [instance_market_options.value.spot_options] : [] + for_each = try([instance_market_options.value.spot_options], []) + + content { + block_duration_minutes = try(spot_options.value.block_duration_minutes, null) + instance_interruption_behavior = try(spot_options.value.instance_interruption_behavior, null) + max_price = try(spot_options.value.max_price, null) + spot_instance_type = try(spot_options.value.spot_instance_type, null) + valid_until = try(spot_options.value.valid_until, null) + } + } + } + } + + dynamic "instance_requirements" { + for_each = length(var.instance_requirements) > 0 ? [var.instance_requirements] : [] + + content { + + dynamic "accelerator_count" { + for_each = try([instance_requirements.value.accelerator_count], []) + + content { + max = try(accelerator_count.value.max, null) + min = try(accelerator_count.value.min, null) + } + } + + accelerator_manufacturers = try(instance_requirements.value.accelerator_manufacturers, []) + accelerator_names = try(instance_requirements.value.accelerator_names, []) + + dynamic "accelerator_total_memory_mib" { + for_each = try([instance_requirements.value.accelerator_total_memory_mib], []) + + content { + max = try(accelerator_total_memory_mib.value.max, null) + min = try(accelerator_total_memory_mib.value.min, null) + } + } + + accelerator_types = try(instance_requirements.value.accelerator_types, []) + bare_metal = try(instance_requirements.value.bare_metal, null) + + dynamic "baseline_ebs_bandwidth_mbps" { + for_each = try([instance_requirements.value.baseline_ebs_bandwidth_mbps], []) + + content { + max = try(baseline_ebs_bandwidth_mbps.value.max, null) + min = try(baseline_ebs_bandwidth_mbps.value.min, null) + } + } + + burstable_performance = try(instance_requirements.value.burstable_performance, null) + cpu_manufacturers = try(instance_requirements.value.cpu_manufacturers, []) + excluded_instance_types = try(instance_requirements.value.excluded_instance_types, []) + instance_generations = try(instance_requirements.value.instance_generations, []) + local_storage = try(instance_requirements.value.local_storage, null) + local_storage_types = try(instance_requirements.value.local_storage_types, []) + + dynamic "memory_gib_per_vcpu" { + for_each = try([instance_requirements.value.memory_gib_per_vcpu], []) + + content { + max = try(memory_gib_per_vcpu.value.max, null) + min = try(memory_gib_per_vcpu.value.min, null) + } + } + + dynamic "memory_mib" { + for_each = [instance_requirements.value.memory_mib] + + content { + max = try(memory_mib.value.max, null) + min = memory_mib.value.min + } + } + + dynamic "network_interface_count" { + for_each = try([instance_requirements.value.network_interface_count], []) + content { - block_duration_minutes = lookup(spot_options.value, "block_duration_minutes", null) - instance_interruption_behavior = lookup(spot_options.value, "instance_interruption_behavior", null) - max_price = lookup(spot_options.value, "max_price", null) - spot_instance_type = lookup(spot_options.value, "spot_instance_type", null) - valid_until = lookup(spot_options.value, "valid_until", null) + max = try(network_interface_count.value.max, null) + min = try(network_interface_count.value.min, null) + } + } + + on_demand_max_price_percentage_over_lowest_price = try(instance_requirements.value.on_demand_max_price_percentage_over_lowest_price, null) + require_hibernate_support = try(instance_requirements.value.require_hibernate_support, null) + spot_max_price_percentage_over_lowest_price = try(instance_requirements.value.spot_max_price_percentage_over_lowest_price, null) + + dynamic "total_local_storage_gb" { + for_each = try([instance_requirements.value.total_local_storage_gb], []) + + content { + max = try(total_local_storage_gb.value.max, null) + min = try(total_local_storage_gb.value.min, null) + } + } + + dynamic "vcpu_count" { + for_each = [instance_requirements.value.vcpu_count] + + content { + max = try(vcpu_count.value.max, null) + min = vcpu_count.value.min } } } } + instance_type = var.instance_type + kernel_id = var.kernel_id + key_name = var.key_name + dynamic "license_specification" { - for_each = length(var.license_specifications) > 0 ? [var.license_specifications] : [] + for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} + content { license_configuration_arn = license_specifications.value.license_configuration_arn } } + dynamic "maintenance_options" { + for_each = length(var.maintenance_options) > 0 ? [var.maintenance_options] : [] + + content { + auto_recovery = try(maintenance_options.value.auto_recovery, null) + } + } + dynamic "metadata_options" { for_each = length(var.metadata_options) > 0 ? [var.metadata_options] : [] + content { - http_endpoint = lookup(metadata_options.value, "http_endpoint", null) - http_tokens = lookup(metadata_options.value, "http_tokens", null) - http_put_response_hop_limit = lookup(metadata_options.value, "http_put_response_hop_limit", null) - http_protocol_ipv6 = lookup(metadata_options.value, "http_protocol_ipv6", null) - instance_metadata_tags = lookup(metadata_options.value, "instance_metadata_tags", null) + http_endpoint = try(metadata_options.value.http_endpoint, null) + http_protocol_ipv6 = try(metadata_options.value.http_protocol_ipv6, null) + http_put_response_hop_limit = try(metadata_options.value.http_put_response_hop_limit, null) + http_tokens = try(metadata_options.value.http_tokens, null) + instance_metadata_tags = try(metadata_options.value.instance_metadata_tags, null) } } dynamic "monitoring" { - for_each = var.enable_monitoring != null ? [1] : [] + for_each = var.enable_monitoring ? [1] : [] + content { enabled = var.enable_monitoring } } + name = var.launch_template_use_name_prefix ? null : local.launch_template_name + name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null + dynamic "network_interfaces" { for_each = var.network_interfaces content { - associate_carrier_ip_address = lookup(network_interfaces.value, "associate_carrier_ip_address", null) - associate_public_ip_address = lookup(network_interfaces.value, "associate_public_ip_address", null) - delete_on_termination = lookup(network_interfaces.value, "delete_on_termination", null) - description = lookup(network_interfaces.value, "description", null) - device_index = lookup(network_interfaces.value, "device_index", null) - interface_type = lookup(network_interfaces.value, "interface_type", null) + associate_carrier_ip_address = try(network_interfaces.value.associate_carrier_ip_address, null) + associate_public_ip_address = try(network_interfaces.value.associate_public_ip_address, null) + delete_on_termination = try(network_interfaces.value.delete_on_termination, null) + description = try(network_interfaces.value.description, null) + device_index = try(network_interfaces.value.device_index, null) + interface_type = try(network_interfaces.value.interface_type, null) + ipv4_address_count = try(network_interfaces.value.ipv4_address_count, null) ipv4_addresses = try(network_interfaces.value.ipv4_addresses, []) - ipv4_address_count = lookup(network_interfaces.value, "ipv4_address_count", null) + ipv4_prefix_count = try(network_interfaces.value.ipv4_prefix_count, null) + ipv4_prefixes = try(network_interfaces.value.ipv4_prefixes, null) + ipv6_address_count = try(network_interfaces.value.ipv6_address_count, null) ipv6_addresses = try(network_interfaces.value.ipv6_addresses, []) - ipv6_address_count = lookup(network_interfaces.value, "ipv6_address_count", null) - network_interface_id = lookup(network_interfaces.value, "network_interface_id", null) - private_ip_address = lookup(network_interfaces.value, "private_ip_address", null) - security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) - subnet_id = lookup(network_interfaces.value, "subnet_id", null) + ipv6_prefix_count = try(network_interfaces.value.ipv6_prefix_count, null) + ipv6_prefixes = try(network_interfaces.value.ipv6_prefixes, []) + network_card_index = try(network_interfaces.value.network_card_index, null) + network_interface_id = try(network_interfaces.value.network_interface_id, null) + private_ip_address = try(network_interfaces.value.private_ip_address, null) + # Ref: https://github.com/hashicorp/terraform-provider-aws/issues/4570 + security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) + subnet_id = try(network_interfaces.value.subnet_id, null) } } dynamic "placement" { for_each = length(var.placement) > 0 ? [var.placement] : [] + + content { + affinity = try(placement.value.affinity, null) + availability_zone = try(placement.value.availability_zone, null) + group_name = try(placement.value.group_name, null) + host_id = try(placement.value.host_id, null) + host_resource_group_arn = try(placement.value.host_resource_group_arn, null) + partition_number = try(placement.value.partition_number, null) + spread_domain = try(placement.value.spread_domain, null) + tenancy = try(placement.value.tenancy, null) + } + } + + dynamic "private_dns_name_options" { + for_each = length(var.private_dns_name_options) > 0 ? [var.private_dns_name_options] : [] + content { - affinity = lookup(placement.value, "affinity", null) - availability_zone = lookup(placement.value, "availability_zone", null) - group_name = lookup(placement.value, "group_name", null) - host_id = lookup(placement.value, "host_id", null) - spread_domain = lookup(placement.value, "spread_domain", null) - tenancy = lookup(placement.value, "tenancy", null) - partition_number = lookup(placement.value, "partition_number", null) + enable_resource_name_dns_aaaa_record = try(private_dns_name_options.value.enable_resource_name_dns_aaaa_record, null) + enable_resource_name_dns_a_record = try(private_dns_name_options.value.enable_resource_name_dns_a_record, null) + hostname_type = try(private_dns_name_options.value.hostname_type, null) } } + ram_disk_id = var.ram_disk_id + dynamic "tag_specifications" { for_each = toset(["instance", "volume", "network-interface"]) + content { resource_type = tag_specifications.key tags = merge(var.tags, { Name = var.name }, var.launch_template_tags) } } - lifecycle { - create_before_destroy = true - } + update_default_version = var.update_launch_template_default_version + user_data = module.user_data.user_data + vpc_security_group_ids = length(var.network_interfaces) > 0 ? [] : local.security_group_ids + + tags = var.tags - # Prevent premature access of security group roles and policies by pods that + # Prevent premature access of policies by pods that # require permissions on create/destroy that depend on nodes depends_on = [ - aws_security_group_rule.this, aws_iam_role_policy_attachment.this, ] - tags = var.tags + lifecycle { + create_before_destroy = true + } } ################################################################################ @@ -257,7 +394,7 @@ resource "aws_launch_template" "this" { ################################################################################ locals { - launch_template_name = try(aws_launch_template.this[0].name, var.launch_template_name) + launch_template_id = var.create && var.create_launch_template ? aws_launch_template.this[0].id : var.launch_template_id # Change order to allow users to set version priority before using defaults launch_template_version = coalesce(var.launch_template_version, try(aws_launch_template.this[0].default_version, "$Default")) } @@ -265,123 +402,215 @@ locals { resource "aws_autoscaling_group" "this" { count = var.create && var.create_autoscaling_group ? 1 : 0 - name = var.use_name_prefix ? null : var.name - name_prefix = var.use_name_prefix ? "${var.name}-" : null - - dynamic "launch_template" { - for_each = var.use_mixed_instances_policy ? [] : [1] - - content { - name = local.launch_template_name - version = local.launch_template_version - } - } - - availability_zones = var.availability_zones - vpc_zone_identifier = var.subnet_ids - - min_size = var.min_size - max_size = var.max_size - desired_capacity = var.desired_size + availability_zones = var.availability_zones capacity_rebalance = var.capacity_rebalance - min_elb_capacity = var.min_elb_capacity - wait_for_elb_capacity = var.wait_for_elb_capacity - wait_for_capacity_timeout = var.wait_for_capacity_timeout + context = var.context default_cooldown = var.default_cooldown - protect_from_scale_in = var.protect_from_scale_in - - target_group_arns = var.target_group_arns - placement_group = var.placement_group - health_check_type = var.health_check_type + default_instance_warmup = var.default_instance_warmup + desired_capacity = var.desired_size + enabled_metrics = var.enabled_metrics + force_delete = var.force_delete + force_delete_warm_pool = var.force_delete_warm_pool health_check_grace_period = var.health_check_grace_period - - force_delete = var.force_delete - termination_policies = var.termination_policies - suspended_processes = var.suspended_processes - max_instance_lifetime = var.max_instance_lifetime - - enabled_metrics = var.enabled_metrics - metrics_granularity = var.metrics_granularity - service_linked_role_arn = var.service_linked_role_arn + health_check_type = var.health_check_type dynamic "initial_lifecycle_hook" { for_each = var.initial_lifecycle_hooks + content { - name = initial_lifecycle_hook.value.name - default_result = lookup(initial_lifecycle_hook.value, "default_result", null) - heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) + default_result = try(initial_lifecycle_hook.value.default_result, null) + heartbeat_timeout = try(initial_lifecycle_hook.value.heartbeat_timeout, null) lifecycle_transition = initial_lifecycle_hook.value.lifecycle_transition - notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) - notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) - role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) + name = initial_lifecycle_hook.value.name + notification_metadata = try(initial_lifecycle_hook.value.notification_metadata, null) + notification_target_arn = try(initial_lifecycle_hook.value.notification_target_arn, null) + role_arn = try(initial_lifecycle_hook.value.role_arn, null) } } dynamic "instance_refresh" { for_each = length(var.instance_refresh) > 0 ? [var.instance_refresh] : [] - content { - strategy = instance_refresh.value.strategy - triggers = lookup(instance_refresh.value, "triggers", null) + content { dynamic "preferences" { - for_each = length(lookup(instance_refresh.value, "preferences", {})) > 0 ? [instance_refresh.value.preferences] : [] + for_each = try([instance_refresh.value.preferences], []) + content { - instance_warmup = lookup(preferences.value, "instance_warmup", null) - min_healthy_percentage = lookup(preferences.value, "min_healthy_percentage", null) - checkpoint_delay = lookup(preferences.value, "checkpoint_delay", null) - checkpoint_percentages = lookup(preferences.value, "checkpoint_percentages", null) + checkpoint_delay = try(preferences.value.checkpoint_delay, null) + checkpoint_percentages = try(preferences.value.checkpoint_percentages, null) + instance_warmup = try(preferences.value.instance_warmup, null) + min_healthy_percentage = try(preferences.value.min_healthy_percentage, null) + skip_matching = try(preferences.value.skip_matching, null) } } + + strategy = instance_refresh.value.strategy + triggers = try(instance_refresh.value.triggers, null) + } + } + + dynamic "launch_template" { + for_each = var.use_mixed_instances_policy ? [] : [1] + + content { + id = local.launch_template_id + version = local.launch_template_version } } + max_instance_lifetime = var.max_instance_lifetime + max_size = var.max_size + metrics_granularity = var.metrics_granularity + min_elb_capacity = var.min_elb_capacity + min_size = var.min_size + dynamic "mixed_instances_policy" { for_each = var.use_mixed_instances_policy ? [var.mixed_instances_policy] : [] + content { dynamic "instances_distribution" { for_each = try([mixed_instances_policy.value.instances_distribution], []) + content { - on_demand_allocation_strategy = lookup(instances_distribution.value, "on_demand_allocation_strategy", null) - on_demand_base_capacity = lookup(instances_distribution.value, "on_demand_base_capacity", null) - on_demand_percentage_above_base_capacity = lookup(instances_distribution.value, "on_demand_percentage_above_base_capacity", null) - spot_allocation_strategy = lookup(instances_distribution.value, "spot_allocation_strategy", null) - spot_instance_pools = lookup(instances_distribution.value, "spot_instance_pools", null) - spot_max_price = lookup(instances_distribution.value, "spot_max_price", null) + on_demand_allocation_strategy = try(instances_distribution.value.on_demand_allocation_strategy, null) + on_demand_base_capacity = try(instances_distribution.value.on_demand_base_capacity, null) + on_demand_percentage_above_base_capacity = try(instances_distribution.value.on_demand_percentage_above_base_capacity, null) + spot_allocation_strategy = try(instances_distribution.value.spot_allocation_strategy, null) + spot_instance_pools = try(instances_distribution.value.spot_instance_pools, null) + spot_max_price = try(instances_distribution.value.spot_max_price, null) } } launch_template { launch_template_specification { - launch_template_name = local.launch_template_name - version = local.launch_template_version + launch_template_id = local.launch_template_id + version = local.launch_template_version } dynamic "override" { for_each = try(mixed_instances_policy.value.override, []) + content { - instance_type = lookup(override.value, "instance_type", null) - weighted_capacity = lookup(override.value, "weighted_capacity", null) + dynamic "instance_requirements" { + for_each = try([override.value.instance_requirements], []) + + content { + + dynamic "accelerator_count" { + for_each = try([instance_requirements.value.accelerator_count], []) + + content { + max = try(accelerator_count.value.max, null) + min = try(accelerator_count.value.min, null) + } + } + + accelerator_manufacturers = try(instance_requirements.value.accelerator_manufacturers, []) + accelerator_names = try(instance_requirements.value.accelerator_names, []) + + dynamic "accelerator_total_memory_mib" { + for_each = try([instance_requirements.value.accelerator_total_memory_mib], []) + + content { + max = try(accelerator_total_memory_mib.value.max, null) + min = try(accelerator_total_memory_mib.value.min, null) + } + } + + accelerator_types = try(instance_requirements.value.accelerator_types, []) + bare_metal = try(instance_requirements.value.bare_metal, null) + + dynamic "baseline_ebs_bandwidth_mbps" { + for_each = try([instance_requirements.value.baseline_ebs_bandwidth_mbps], []) + + content { + max = try(baseline_ebs_bandwidth_mbps.value.max, null) + min = try(baseline_ebs_bandwidth_mbps.value.min, null) + } + } + + burstable_performance = try(instance_requirements.value.burstable_performance, null) + cpu_manufacturers = try(instance_requirements.value.cpu_manufacturers, []) + excluded_instance_types = try(instance_requirements.value.excluded_instance_types, []) + instance_generations = try(instance_requirements.value.instance_generations, []) + local_storage = try(instance_requirements.value.local_storage, null) + local_storage_types = try(instance_requirements.value.local_storage_types, []) + + dynamic "memory_gib_per_vcpu" { + for_each = try([instance_requirements.value.memory_gib_per_vcpu], []) + + content { + max = try(memory_gib_per_vcpu.value.max, null) + min = try(memory_gib_per_vcpu.value.min, null) + } + } + + dynamic "memory_mib" { + for_each = [instance_requirements.value.memory_mib] + + content { + max = try(memory_mib.value.max, null) + min = memory_mib.value.min + } + } + + dynamic "network_interface_count" { + for_each = try([instance_requirements.value.network_interface_count], []) + + content { + max = try(network_interface_count.value.max, null) + min = try(network_interface_count.value.min, null) + } + } + + on_demand_max_price_percentage_over_lowest_price = try(instance_requirements.value.on_demand_max_price_percentage_over_lowest_price, null) + require_hibernate_support = try(instance_requirements.value.require_hibernate_support, null) + spot_max_price_percentage_over_lowest_price = try(instance_requirements.value.spot_max_price_percentage_over_lowest_price, null) + + dynamic "total_local_storage_gb" { + for_each = try([instance_requirements.value.total_local_storage_gb], []) + + content { + max = try(total_local_storage_gb.value.max, null) + min = try(total_local_storage_gb.value.min, null) + } + } + + dynamic "vcpu_count" { + for_each = [instance_requirements.value.vcpu_count] + + content { + max = try(vcpu_count.value.max, null) + min = vcpu_count.value.min + } + } + } + } + + instance_type = try(override.value.instance_type, null) dynamic "launch_template_specification" { - for_each = length(lookup(override.value, "launch_template_specification", {})) > 0 ? override.value.launch_template_specification : [] + for_each = try([override.value.launch_template_specification], []) + content { - launch_template_id = lookup(launch_template_specification.value, "launch_template_id", null) + launch_template_id = try(launch_template_specification.value.launch_template_id, null) + version = try(launch_template_specification.value.version, null) } } + + weighted_capacity = try(override.value.weighted_capacity, null) } } } } } - dynamic "warm_pool" { - for_each = length(var.warm_pool) > 0 ? [var.warm_pool] : [] - content { - pool_state = lookup(warm_pool.value, "pool_state", null) - min_size = lookup(warm_pool.value, "min_size", null) - max_group_prepared_capacity = lookup(warm_pool.value, "max_group_prepared_capacity", null) - } - } + name = var.use_name_prefix ? null : var.name + name_prefix = var.use_name_prefix ? "${var.name}-" : null + placement_group = var.placement_group + protect_from_scale_in = var.protect_from_scale_in + service_linked_role_arn = var.service_linked_role_arn + suspended_processes = var.suspended_processes dynamic "tag" { for_each = merge( @@ -390,7 +619,7 @@ resource "aws_autoscaling_group" "this" { "kubernetes.io/cluster/${var.cluster_name}" = "owned" "k8s.io/cluster/${var.cluster_name}" = "owned" }, - var.use_default_tags ? merge(data.aws_default_tags.current.tags, var.tags) : var.tags + var.tags ) content { @@ -410,6 +639,30 @@ resource "aws_autoscaling_group" "this" { } } + target_group_arns = var.target_group_arns + termination_policies = var.termination_policies + vpc_zone_identifier = var.subnet_ids + wait_for_capacity_timeout = var.wait_for_capacity_timeout + wait_for_elb_capacity = var.wait_for_elb_capacity + + dynamic "warm_pool" { + for_each = length(var.warm_pool) > 0 ? [var.warm_pool] : [] + + content { + dynamic "instance_reuse_policy" { + for_each = try([warm_pool.value.instance_reuse_policy], []) + + content { + reuse_on_scale_in = try(instance_reuse_policy.value.reuse_on_scale_in, null) + } + } + + max_group_prepared_capacity = try(warm_pool.value.max_group_prepared_capacity, null) + min_size = try(warm_pool.value.min_size, null) + pool_state = try(warm_pool.value.pool_state, null) + } + } + timeouts { delete = var.delete_timeout } @@ -432,70 +685,16 @@ resource "aws_autoscaling_schedule" "this" { scheduled_action_name = each.key autoscaling_group_name = aws_autoscaling_group.this[0].name - min_size = lookup(each.value, "min_size", null) - max_size = lookup(each.value, "max_size", null) - desired_capacity = lookup(each.value, "desired_size", null) - start_time = lookup(each.value, "start_time", null) - end_time = lookup(each.value, "end_time", null) - time_zone = lookup(each.value, "time_zone", null) + min_size = try(each.value.min_size, null) + max_size = try(each.value.max_size, null) + desired_capacity = try(each.value.desired_size, null) + start_time = try(each.value.start_time, null) + end_time = try(each.value.end_time, null) + time_zone = try(each.value.time_zone, null) # [Minute] [Hour] [Day_of_Month] [Month_of_Year] [Day_of_Week] # Cron examples: https://crontab.guru/examples.html - recurrence = lookup(each.value, "recurrence", null) -} - -################################################################################ -# Security Group -################################################################################ - -locals { - security_group_name = coalesce(var.security_group_name, "${var.name}-node-group") - create_security_group = var.create && var.create_security_group -} - -resource "aws_security_group" "this" { - count = local.create_security_group ? 1 : 0 - - name = var.security_group_use_name_prefix ? null : local.security_group_name - name_prefix = var.security_group_use_name_prefix ? "${local.security_group_name}-" : null - description = var.security_group_description - vpc_id = var.vpc_id - - tags = merge( - var.tags, - { - "Name" = local.security_group_name - }, - var.security_group_tags - ) - - # https://github.com/hashicorp/terraform-provider-aws/issues/2445 - # https://github.com/hashicorp/terraform-provider-aws/issues/9692 - lifecycle { - create_before_destroy = true - } -} - -resource "aws_security_group_rule" "this" { - for_each = { for k, v in var.security_group_rules : k => v if local.create_security_group } - - # Required - security_group_id = aws_security_group.this[0].id - protocol = each.value.protocol - from_port = each.value.from_port - to_port = each.value.to_port - type = each.value.type - - # Optional - description = try(each.value.description, null) - cidr_blocks = try(each.value.cidr_blocks, null) - ipv6_cidr_blocks = try(each.value.ipv6_cidr_blocks, null) - prefix_list_ids = try(each.value.prefix_list_ids, []) - self = try(each.value.self, null) - source_security_group_id = try( - each.value.source_security_group_id, - try(each.value.source_cluster_security_group, false) ? var.cluster_security_group_id : null - ) + recurrence = try(each.value.recurrence, null) } ################################################################################ @@ -503,11 +702,9 @@ resource "aws_security_group_rule" "this" { ################################################################################ locals { - iam_role_name = coalesce(var.iam_role_name, "${var.name}-node-group") - + iam_role_name = coalesce(var.iam_role_name, "${var.name}-node-group") iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" - - cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } data "aws_iam_policy_document" "assume_role_policy" { @@ -540,17 +737,23 @@ resource "aws_iam_role" "this" { } resource "aws_iam_role_policy_attachment" "this" { - for_each = var.create && var.create_iam_instance_profile ? toset(compact(distinct(concat([ + for_each = { for k, v in toset(compact([ "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", var.iam_role_attach_cni_policy ? local.cni_policy : "", - ], var.iam_role_additional_policies)))) : toset([]) + ])) : k => v if var.create && var.create_iam_instance_profile } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_additional_policies : k => v if var.create && var.create_iam_instance_profile } policy_arn = each.value role = aws_iam_role.this[0].name } -# Only self-managed node group requires instance profile resource "aws_iam_instance_profile" "this" { count = var.create && var.create_iam_instance_profile ? 1 : 0 @@ -560,9 +763,9 @@ resource "aws_iam_instance_profile" "this" { name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null path = var.iam_role_path + tags = merge(var.tags, var.iam_role_tags) + lifecycle { create_before_destroy = true } - - tags = merge(var.tags, var.iam_role_tags) } diff --git a/modules/self-managed-node-group/outputs.tf b/modules/self-managed-node-group/outputs.tf index e9f52db14f..c9c9b75254 100644 --- a/modules/self-managed-node-group/outputs.tf +++ b/modules/self-managed-node-group/outputs.tf @@ -4,22 +4,22 @@ output "launch_template_id" { description = "The ID of the launch template" - value = try(aws_launch_template.this[0].id, "") + value = try(aws_launch_template.this[0].id, null) } output "launch_template_arn" { description = "The ARN of the launch template" - value = try(aws_launch_template.this[0].arn, "") + value = try(aws_launch_template.this[0].arn, null) } output "launch_template_latest_version" { description = "The latest version of the launch template" - value = try(aws_launch_template.this[0].latest_version, "") + value = try(aws_launch_template.this[0].latest_version, null) } output "launch_template_name" { description = "The name of the launch template" - value = try(aws_launch_template.this[0].name, "") + value = try(aws_launch_template.this[0].name, null) } ################################################################################ @@ -28,57 +28,57 @@ output "launch_template_name" { output "autoscaling_group_arn" { description = "The ARN for this autoscaling group" - value = try(aws_autoscaling_group.this[0].arn, "") + value = try(aws_autoscaling_group.this[0].arn, null) } output "autoscaling_group_id" { description = "The autoscaling group id" - value = try(aws_autoscaling_group.this[0].id, "") + value = try(aws_autoscaling_group.this[0].id, null) } output "autoscaling_group_name" { description = "The autoscaling group name" - value = try(aws_autoscaling_group.this[0].name, "") + value = try(aws_autoscaling_group.this[0].name, null) } output "autoscaling_group_min_size" { description = "The minimum size of the autoscaling group" - value = try(aws_autoscaling_group.this[0].min_size, "") + value = try(aws_autoscaling_group.this[0].min_size, null) } output "autoscaling_group_max_size" { description = "The maximum size of the autoscaling group" - value = try(aws_autoscaling_group.this[0].max_size, "") + value = try(aws_autoscaling_group.this[0].max_size, null) } output "autoscaling_group_desired_capacity" { description = "The number of Amazon EC2 instances that should be running in the group" - value = try(aws_autoscaling_group.this[0].desired_capacity, "") + value = try(aws_autoscaling_group.this[0].desired_capacity, null) } output "autoscaling_group_default_cooldown" { description = "Time between a scaling activity and the succeeding scaling activity" - value = try(aws_autoscaling_group.this[0].default_cooldown, "") + value = try(aws_autoscaling_group.this[0].default_cooldown, null) } output "autoscaling_group_health_check_grace_period" { description = "Time after instance comes into service before checking health" - value = try(aws_autoscaling_group.this[0].health_check_grace_period, "") + value = try(aws_autoscaling_group.this[0].health_check_grace_period, null) } output "autoscaling_group_health_check_type" { description = "EC2 or ELB. Controls how health checking is done" - value = try(aws_autoscaling_group.this[0].health_check_type, "") + value = try(aws_autoscaling_group.this[0].health_check_type, null) } output "autoscaling_group_availability_zones" { description = "The availability zones of the autoscaling group" - value = try(aws_autoscaling_group.this[0].availability_zones, "") + value = try(aws_autoscaling_group.this[0].availability_zones, null) } output "autoscaling_group_vpc_zone_identifier" { description = "The VPC zone identifier" - value = try(aws_autoscaling_group.this[0].vpc_zone_identifier, "") + value = try(aws_autoscaling_group.this[0].vpc_zone_identifier, null) } ################################################################################ @@ -90,37 +90,23 @@ output "autoscaling_group_schedule_arns" { value = { for k, v in aws_autoscaling_schedule.this : k => v.arn } } -################################################################################ -# Security Group -################################################################################ - -output "security_group_arn" { - description = "Amazon Resource Name (ARN) of the security group" - value = try(aws_security_group.this[0].arn, "") -} - -output "security_group_id" { - description = "ID of the security group" - value = try(aws_security_group.this[0].id, "") -} - ################################################################################ # IAM Role ################################################################################ output "iam_role_name" { description = "The name of the IAM role" - value = try(aws_iam_role.this[0].name, "") + value = try(aws_iam_role.this[0].name, null) } output "iam_role_arn" { description = "The Amazon Resource Name (ARN) specifying the IAM role" - value = try(aws_iam_role.this[0].arn, "") + value = try(aws_iam_role.this[0].arn, null) } output "iam_role_unique_id" { description = "Stable and unique string identifying the IAM role" - value = try(aws_iam_role.this[0].unique_id, "") + value = try(aws_iam_role.this[0].unique_id, null) } ################################################################################ @@ -134,12 +120,12 @@ output "iam_instance_profile_arn" { output "iam_instance_profile_id" { description = "Instance profile's ID" - value = try(aws_iam_instance_profile.this[0].id, "") + value = try(aws_iam_instance_profile.this[0].id, null) } output "iam_instance_profile_unique" { description = "Stable and unique string identifying the IAM instance profile" - value = try(aws_iam_instance_profile.this[0].unique_id, "") + value = try(aws_iam_instance_profile.this[0].unique_id, null) } ################################################################################ @@ -153,10 +139,10 @@ output "platform" { output "image_id" { description = "ID of the image" - value = try(aws_launch_template.this[0].image_id, "") + value = try(aws_launch_template.this[0].image_id, null) } output "user_data" { description = "Base64 encoded user data" - value = try(module.user_data.user_data, "") + value = try(module.user_data.user_data, null) } diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 3734d9068c..c2850b659e 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -72,8 +72,14 @@ variable "create_launch_template" { default = true } +variable "launch_template_id" { + description = "The ID of an existing launch template to use. Required when `create_launch_template` = `false`" + type = string + default = "" +} + variable "launch_template_name" { - description = "Launch template name - either to be created (`var.create_launch_template` = `true`) or existing (`var.create_launch_template` = `false`)" + description = "Name of launch template to be created" type = string default = null } @@ -152,7 +158,7 @@ variable "credit_specification" { variable "elastic_gpu_specifications" { description = "The elastic GPU to attach to the instance" - type = map(string) + type = any default = {} } @@ -180,9 +186,15 @@ variable "instance_market_options" { default = {} } +variable "maintenance_options" { + description = "The maintenance options for the instance" + type = any + default = {} +} + variable "license_specifications" { - description = "A list of license specifications to associate with" - type = map(string) + description = "A map of license specifications to associate with" + type = any default = {} } @@ -198,6 +210,12 @@ variable "placement" { default = {} } +variable "private_dns_name_options" { + description = "The options for the instance hostname. The default values are inherited from the subnet" + type = map(string) + default = {} +} + variable "ebs_optimized" { description = "If true, the launched EC2 instance will be EBS-optimized" type = bool @@ -216,6 +234,12 @@ variable "cluster_version" { default = null } +variable "instance_requirements" { + description = "The attribute requirements for the type of instance. If present then `instance_type` cannot be present" + type = any + default = {} +} + variable "instance_type" { description = "The type of the instance to launch" type = string @@ -320,6 +344,12 @@ variable "desired_size" { default = 1 } +variable "context" { + description = "Reserved" + type = string + default = null +} + variable "capacity_rebalance" { description = "Indicates whether capacity rebalance is enabled" type = bool @@ -350,6 +380,12 @@ variable "default_cooldown" { default = null } +variable "default_instance_warmup" { + description = "Amount of time, in seconds, until a newly launched instance can contribute to the Amazon CloudWatch metrics. This delay lets an instance finish initializing before Amazon EC2 Auto Scaling aggregates instance metrics, resulting in more reliable usage data" + type = number + default = null +} + variable "protect_from_scale_in" { description = "Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events." type = bool @@ -386,6 +422,12 @@ variable "force_delete" { default = null } +variable "force_delete_warm_pool" { + description = "Allows deleting the Auto Scaling Group without waiting for all instances in the warm pool to terminate" + type = bool + default = null +} + variable "termination_policies" { description = "A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default`" type = list(string) @@ -431,7 +473,12 @@ variable "initial_lifecycle_hooks" { variable "instance_refresh" { description = "If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated" type = any - default = {} + default = { + strategy = "Rolling" + preferences = { + min_healthy_percentage = 66 + } + } } variable "use_mixed_instances_policy" { @@ -458,12 +505,6 @@ variable "delete_timeout" { default = null } -variable "use_default_tags" { - description = "Enables/disables the use of provider default tags in the tag_specifications of the Auto Scaling group" - type = bool - default = false -} - variable "autoscaling_group_tags" { description = "A map of additional tags to add to the autoscaling group created. Tags are applied to the autoscaling group only and are NOT propagated to instances" type = map(string) @@ -486,58 +527,6 @@ variable "schedules" { default = {} } -################################################################################ -# Security Group -################################################################################ - -variable "create_security_group" { - description = "Determines whether to create a security group" - type = bool - default = true -} - -variable "security_group_name" { - description = "Name to use on security group created" - type = string - default = null -} - -variable "security_group_use_name_prefix" { - description = "Determines whether the security group name (`security_group_name`) is used as a prefix" - type = bool - default = true -} - -variable "security_group_description" { - description = "Description for the security group created" - type = string - default = "EKS self-managed node group security group" -} - -variable "vpc_id" { - description = "ID of the VPC where the security group/nodes will be provisioned" - type = string - default = null -} - -variable "security_group_rules" { - description = "List of security group rules to add to the security group created" - type = any - default = {} -} - -variable "cluster_security_group_id" { - description = "Cluster control plane security group ID" - type = string - default = null -} - -variable "security_group_tags" { - description = "A map of additional tags to add to the security group created" - type = map(string) - default = {} -} - ################################################################################ # IAM Role ################################################################################ @@ -598,8 +587,8 @@ variable "iam_role_attach_cni_policy" { variable "iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" - type = list(string) - default = [] + type = map(string) + default = {} } variable "iam_role_tags" { diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 22e8d7265f..325eee94e1 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72" + version = ">= 4.45" } } } diff --git a/node_groups.tf b/node_groups.tf index 402191f162..36f071610c 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -4,6 +4,19 @@ locals { http_tokens = "required" http_put_response_hop_limit = 2 } + + # EKS managed node group + default_update_config = { + max_unavailable_percentage = 33 + } + + # Self-managed node group + default_instance_refresh = { + strategy = "Rolling" + preferences = { + min_healthy_percentage = 66 + } + } } ################################################################################ @@ -59,14 +72,6 @@ locals { node_security_group_id = local.create_node_sg ? aws_security_group.node[0].id : var.node_security_group_id node_security_group_rules = { - egress_cluster_443 = { - description = "Node groups to cluster API" - protocol = "tcp" - from_port = 443 - to_port = 443 - type = "egress" - source_cluster_security_group = true - } ingress_cluster_443 = { description = "Cluster API to node groups" protocol = "tcp" @@ -91,58 +96,51 @@ locals { type = "ingress" self = true } - egress_self_coredns_tcp = { - description = "Node to node CoreDNS" - protocol = "tcp" - from_port = 53 - to_port = 53 - type = "egress" - self = true - } ingress_self_coredns_udp = { - description = "Node to node CoreDNS" + description = "Node to node CoreDNS UDP" protocol = "udp" from_port = 53 to_port = 53 type = "ingress" self = true } - egress_self_coredns_udp = { - description = "Node to node CoreDNS" - protocol = "udp" - from_port = 53 - to_port = 53 - type = "egress" + } + + node_secuirty_group_recommended_rules = { for k, v in { + ingress_nodes_ephemeral = { + description = "Node to node ingress on ephemeral ports" + protocol = "tcp" + from_port = 1025 + to_port = 65535 + type = "ingress" self = true } - egress_https = { - description = "Egress all HTTPS to internet" - protocol = "tcp" - from_port = 443 - to_port = 443 - type = "egress" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null + ingress_cluster_8443_webhook = { + description = "Cluster API to node 8443/tcp webhook" + protocol = "tcp" + from_port = 8443 + to_port = 8443 + type = "ingress" + source_cluster_security_group = true } - egress_ntp_tcp = { - description = "Egress NTP/TCP to internet" - protocol = "tcp" - from_port = 123 - to_port = 123 - type = "egress" - cidr_blocks = var.node_security_group_ntp_ipv4_cidr_block - ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? var.node_security_group_ntp_ipv6_cidr_block : null + ingress_cluster_9443_webhook = { + description = "Cluster API to node 9443/tcp webhook" + protocol = "tcp" + from_port = 9443 + to_port = 9443 + type = "ingress" + source_cluster_security_group = true } - egress_ntp_udp = { - description = "Egress NTP/UDP to internet" - protocol = "udp" - from_port = 123 - to_port = 123 + egress_all = { + description = "Allow all egress" + protocol = "-1" + from_port = 0 + to_port = 0 type = "egress" - cidr_blocks = var.node_security_group_ntp_ipv4_cidr_block - ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? var.node_security_group_ntp_ipv6_cidr_block : null + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null } - } + } : k => v if var.node_security_group_enable_recommended_rules } } resource "aws_security_group" "node" { @@ -168,7 +166,11 @@ resource "aws_security_group" "node" { } resource "aws_security_group_rule" "node" { - for_each = { for k, v in merge(local.node_security_group_rules, var.node_security_group_additional_rules) : k => v if local.create_node_sg } + for_each = { for k, v in merge( + local.node_security_group_rules, + local.node_secuirty_group_recommended_rules, + var.node_security_group_additional_rules, + ) : k => v if local.create_node_sg } # Required security_group_id = aws_security_group.node[0].id @@ -178,15 +180,13 @@ resource "aws_security_group_rule" "node" { type = each.value.type # Optional - description = try(each.value.description, null) - cidr_blocks = try(each.value.cidr_blocks, null) - ipv6_cidr_blocks = try(each.value.ipv6_cidr_blocks, null) - prefix_list_ids = try(each.value.prefix_list_ids, []) - self = try(each.value.self, null) - source_security_group_id = try( - each.value.source_security_group_id, - try(each.value.source_cluster_security_group, false) ? local.cluster_security_group_id : null - ) + description = lookup(each.value, "description", null) + cidr_blocks = lookup(each.value, "cidr_blocks", null) + ipv6_cidr_blocks = lookup(each.value, "ipv6_cidr_blocks", null) + prefix_list_ids = lookup(each.value, "prefix_list_ids", []) + self = lookup(each.value, "self", null) + source_security_group_id = lookup(each.value, "source_security_group_id", + lookup(each.value, "source_cluster_security_group", false)) ? local.cluster_security_group_id : null } ################################################################################ @@ -196,7 +196,7 @@ resource "aws_security_group_rule" "node" { module "fargate_profile" { source = "./modules/fargate-profile" - for_each = { for k, v in var.fargate_profiles : k => v if var.create } + for_each = { for k, v in var.fargate_profiles : k => v if var.create && !local.create_outposts_local_cluster } create = try(each.value.create, true) @@ -218,7 +218,9 @@ module "fargate_profile" { iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.fargate_profile_defaults.iam_role_permissions_boundary, null) iam_role_tags = try(each.value.iam_role_tags, var.fargate_profile_defaults.iam_role_tags, {}) iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.fargate_profile_defaults.iam_role_attach_cni_policy, true) - iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.fargate_profile_defaults.iam_role_additional_policies, []) + # To better understand why this `lookup()` logic is required, see: + # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 + iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.fargate_profile_defaults, "iam_role_additional_policies", {})) tags = merge(var.tags, try(each.value.tags, var.fargate_profile_defaults.tags, {})) } @@ -230,14 +232,13 @@ module "fargate_profile" { module "eks_managed_node_group" { source = "./modules/eks-managed-node-group" - for_each = { for k, v in var.eks_managed_node_groups : k => v if var.create } + for_each = { for k, v in var.eks_managed_node_groups : k => v if var.create && !local.create_outposts_local_cluster } create = try(each.value.create, true) - cluster_name = aws_eks_cluster.this[0].name - cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, aws_eks_cluster.this[0].version) - cluster_security_group_id = local.cluster_security_group_id - cluster_ip_family = var.cluster_ip_family + cluster_name = aws_eks_cluster.this[0].name + cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, aws_eks_cluster.this[0].version) + cluster_ip_family = var.cluster_ip_family # EKS Managed Node Group name = try(each.value.name, each.key) @@ -261,7 +262,7 @@ module "eks_managed_node_group" { remote_access = try(each.value.remote_access, var.eks_managed_node_group_defaults.remote_access, {}) taints = try(each.value.taints, var.eks_managed_node_group_defaults.taints, {}) - update_config = try(each.value.update_config, var.eks_managed_node_group_defaults.update_config, {}) + update_config = try(each.value.update_config, var.eks_managed_node_group_defaults.update_config, local.default_update_config) timeouts = try(each.value.timeouts, var.eks_managed_node_group_defaults.timeouts, {}) # User data @@ -276,20 +277,22 @@ module "eks_managed_node_group" { user_data_template_path = try(each.value.user_data_template_path, var.eks_managed_node_group_defaults.user_data_template_path, "") # Launch Template - create_launch_template = try(each.value.create_launch_template, var.eks_managed_node_group_defaults.create_launch_template, true) - launch_template_name = try(each.value.launch_template_name, var.eks_managed_node_group_defaults.launch_template_name, each.key) - launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.eks_managed_node_group_defaults.launch_template_use_name_prefix, true) - launch_template_version = try(each.value.launch_template_version, var.eks_managed_node_group_defaults.launch_template_version, null) - launch_template_description = try(each.value.launch_template_description, var.eks_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} EKS managed node group") - launch_template_tags = try(each.value.launch_template_tags, var.eks_managed_node_group_defaults.launch_template_tags, {}) - - ebs_optimized = try(each.value.ebs_optimized, var.eks_managed_node_group_defaults.ebs_optimized, null) - key_name = try(each.value.key_name, var.eks_managed_node_group_defaults.key_name, null) + create_launch_template = try(each.value.create_launch_template, var.eks_managed_node_group_defaults.create_launch_template, true) + use_custom_launch_template = try(each.value.use_custom_launch_template, var.eks_managed_node_group_defaults.use_custom_launch_template, true) + launch_template_id = try(each.value.launch_template_id, var.eks_managed_node_group_defaults.launch_template_id, "") + launch_template_name = try(each.value.launch_template_name, var.eks_managed_node_group_defaults.launch_template_name, each.key) + launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.eks_managed_node_group_defaults.launch_template_use_name_prefix, true) + launch_template_version = try(each.value.launch_template_version, var.eks_managed_node_group_defaults.launch_template_version, null) launch_template_default_version = try(each.value.launch_template_default_version, var.eks_managed_node_group_defaults.launch_template_default_version, null) update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.eks_managed_node_group_defaults.update_launch_template_default_version, true) - disable_api_termination = try(each.value.disable_api_termination, var.eks_managed_node_group_defaults.disable_api_termination, null) - kernel_id = try(each.value.kernel_id, var.eks_managed_node_group_defaults.kernel_id, null) - ram_disk_id = try(each.value.ram_disk_id, var.eks_managed_node_group_defaults.ram_disk_id, null) + launch_template_description = try(each.value.launch_template_description, var.eks_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} EKS managed node group") + launch_template_tags = try(each.value.launch_template_tags, var.eks_managed_node_group_defaults.launch_template_tags, {}) + + ebs_optimized = try(each.value.ebs_optimized, var.eks_managed_node_group_defaults.ebs_optimized, null) + key_name = try(each.value.key_name, var.eks_managed_node_group_defaults.key_name, null) + disable_api_termination = try(each.value.disable_api_termination, var.eks_managed_node_group_defaults.disable_api_termination, null) + kernel_id = try(each.value.kernel_id, var.eks_managed_node_group_defaults.kernel_id, null) + ram_disk_id = try(each.value.ram_disk_id, var.eks_managed_node_group_defaults.ram_disk_id, null) block_device_mappings = try(each.value.block_device_mappings, var.eks_managed_node_group_defaults.block_device_mappings, {}) capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.eks_managed_node_group_defaults.capacity_reservation_specification, {}) @@ -304,6 +307,8 @@ module "eks_managed_node_group" { enable_monitoring = try(each.value.enable_monitoring, var.eks_managed_node_group_defaults.enable_monitoring, true) network_interfaces = try(each.value.network_interfaces, var.eks_managed_node_group_defaults.network_interfaces, []) placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, {}) + maintenance_options = try(each.value.maintenance_options, var.eks_managed_node_group_defaults.maintenance_options, {}) + private_dns_name_options = try(each.value.private_dns_name_options, var.eks_managed_node_group_defaults.private_dns_name_options, {}) # IAM role create_iam_role = try(each.value.create_iam_role, var.eks_managed_node_group_defaults.create_iam_role, true) @@ -315,18 +320,13 @@ module "eks_managed_node_group" { iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.eks_managed_node_group_defaults.iam_role_permissions_boundary, null) iam_role_tags = try(each.value.iam_role_tags, var.eks_managed_node_group_defaults.iam_role_tags, {}) iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.eks_managed_node_group_defaults.iam_role_attach_cni_policy, true) - iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.eks_managed_node_group_defaults.iam_role_additional_policies, []) + # To better understand why this `lookup()` logic is required, see: + # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 + iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.eks_managed_node_group_defaults, "iam_role_additional_policies", {})) # Security group vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.eks_managed_node_group_defaults.vpc_security_group_ids, []))) cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, var.eks_managed_node_group_defaults.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null - create_security_group = try(each.value.create_security_group, var.eks_managed_node_group_defaults.create_security_group, true) - security_group_name = try(each.value.security_group_name, var.eks_managed_node_group_defaults.security_group_name, null) - security_group_use_name_prefix = try(each.value.security_group_use_name_prefix, var.eks_managed_node_group_defaults.security_group_use_name_prefix, true) - security_group_description = try(each.value.security_group_description, var.eks_managed_node_group_defaults.security_group_description, "EKS managed node group security group") - vpc_id = try(each.value.vpc_id, var.eks_managed_node_group_defaults.vpc_id, var.vpc_id) - security_group_rules = try(each.value.security_group_rules, var.eks_managed_node_group_defaults.security_group_rules, {}) - security_group_tags = try(each.value.security_group_tags, var.eks_managed_node_group_defaults.security_group_tags, {}) tags = merge(var.tags, try(each.value.tags, var.eks_managed_node_group_defaults.tags, {})) } @@ -362,24 +362,27 @@ module "self_managed_node_group" { wait_for_elb_capacity = try(each.value.wait_for_elb_capacity, var.self_managed_node_group_defaults.wait_for_elb_capacity, null) wait_for_capacity_timeout = try(each.value.wait_for_capacity_timeout, var.self_managed_node_group_defaults.wait_for_capacity_timeout, null) default_cooldown = try(each.value.default_cooldown, var.self_managed_node_group_defaults.default_cooldown, null) + default_instance_warmup = try(each.value.default_instance_warmup, var.self_managed_node_group_defaults.default_instance_warmup, null) protect_from_scale_in = try(each.value.protect_from_scale_in, var.self_managed_node_group_defaults.protect_from_scale_in, null) + context = try(each.value.context, var.self_managed_node_group_defaults.context, null) target_group_arns = try(each.value.target_group_arns, var.self_managed_node_group_defaults.target_group_arns, []) placement_group = try(each.value.placement_group, var.self_managed_node_group_defaults.placement_group, null) health_check_type = try(each.value.health_check_type, var.self_managed_node_group_defaults.health_check_type, null) health_check_grace_period = try(each.value.health_check_grace_period, var.self_managed_node_group_defaults.health_check_grace_period, null) - force_delete = try(each.value.force_delete, var.self_managed_node_group_defaults.force_delete, null) - termination_policies = try(each.value.termination_policies, var.self_managed_node_group_defaults.termination_policies, []) - suspended_processes = try(each.value.suspended_processes, var.self_managed_node_group_defaults.suspended_processes, []) - max_instance_lifetime = try(each.value.max_instance_lifetime, var.self_managed_node_group_defaults.max_instance_lifetime, null) + force_delete = try(each.value.force_delete, var.self_managed_node_group_defaults.force_delete, null) + force_delete_warm_pool = try(each.value.force_delete_warm_pool, var.self_managed_node_group_defaults.force_delete_warm_pool, null) + termination_policies = try(each.value.termination_policies, var.self_managed_node_group_defaults.termination_policies, []) + suspended_processes = try(each.value.suspended_processes, var.self_managed_node_group_defaults.suspended_processes, []) + max_instance_lifetime = try(each.value.max_instance_lifetime, var.self_managed_node_group_defaults.max_instance_lifetime, null) enabled_metrics = try(each.value.enabled_metrics, var.self_managed_node_group_defaults.enabled_metrics, []) metrics_granularity = try(each.value.metrics_granularity, var.self_managed_node_group_defaults.metrics_granularity, null) service_linked_role_arn = try(each.value.service_linked_role_arn, var.self_managed_node_group_defaults.service_linked_role_arn, null) initial_lifecycle_hooks = try(each.value.initial_lifecycle_hooks, var.self_managed_node_group_defaults.initial_lifecycle_hooks, []) - instance_refresh = try(each.value.instance_refresh, var.self_managed_node_group_defaults.instance_refresh, {}) + instance_refresh = try(each.value.instance_refresh, var.self_managed_node_group_defaults.instance_refresh, local.default_instance_refresh) use_mixed_instances_policy = try(each.value.use_mixed_instances_policy, var.self_managed_node_group_defaults.use_mixed_instances_policy, false) mixed_instances_policy = try(each.value.mixed_instances_policy, var.self_managed_node_group_defaults.mixed_instances_policy, null) warm_pool = try(each.value.warm_pool, var.self_managed_node_group_defaults.warm_pool, {}) @@ -388,7 +391,6 @@ module "self_managed_node_group" { schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, {}) delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) - use_default_tags = try(each.value.use_default_tags, var.self_managed_node_group_defaults.use_default_tags, false) autoscaling_group_tags = try(each.value.autoscaling_group_tags, var.self_managed_node_group_defaults.autoscaling_group_tags, {}) # User data @@ -401,12 +403,15 @@ module "self_managed_node_group" { user_data_template_path = try(each.value.user_data_template_path, var.self_managed_node_group_defaults.user_data_template_path, "") # Launch Template - create_launch_template = try(each.value.create_launch_template, var.self_managed_node_group_defaults.create_launch_template, true) - launch_template_name = try(each.value.launch_template_name, var.self_managed_node_group_defaults.launch_template_name, each.key) - launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.self_managed_node_group_defaults.launch_template_use_name_prefix, true) - launch_template_version = try(each.value.launch_template_version, var.self_managed_node_group_defaults.launch_template_version, null) - launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") - launch_template_tags = try(each.value.launch_template_tags, var.self_managed_node_group_defaults.launch_template_tags, {}) + create_launch_template = try(each.value.create_launch_template, var.self_managed_node_group_defaults.create_launch_template, true) + launch_template_id = try(each.value.launch_template_id, var.self_managed_node_group_defaults.launch_template_id, "") + launch_template_name = try(each.value.launch_template_name, var.self_managed_node_group_defaults.launch_template_name, each.key) + launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.self_managed_node_group_defaults.launch_template_use_name_prefix, true) + launch_template_version = try(each.value.launch_template_version, var.self_managed_node_group_defaults.launch_template_version, null) + launch_template_default_version = try(each.value.launch_template_default_version, var.self_managed_node_group_defaults.launch_template_default_version, null) + update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.self_managed_node_group_defaults.update_launch_template_default_version, true) + launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") + launch_template_tags = try(each.value.launch_template_tags, var.self_managed_node_group_defaults.launch_template_tags, {}) ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") @@ -414,12 +419,10 @@ module "self_managed_node_group" { instance_type = try(each.value.instance_type, var.self_managed_node_group_defaults.instance_type, "m6i.large") key_name = try(each.value.key_name, var.self_managed_node_group_defaults.key_name, null) - launch_template_default_version = try(each.value.launch_template_default_version, var.self_managed_node_group_defaults.launch_template_default_version, null) - update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.self_managed_node_group_defaults.update_launch_template_default_version, true) - disable_api_termination = try(each.value.disable_api_termination, var.self_managed_node_group_defaults.disable_api_termination, null) - instance_initiated_shutdown_behavior = try(each.value.instance_initiated_shutdown_behavior, var.self_managed_node_group_defaults.instance_initiated_shutdown_behavior, null) - kernel_id = try(each.value.kernel_id, var.self_managed_node_group_defaults.kernel_id, null) - ram_disk_id = try(each.value.ram_disk_id, var.self_managed_node_group_defaults.ram_disk_id, null) + disable_api_termination = try(each.value.disable_api_termination, var.self_managed_node_group_defaults.disable_api_termination, null) + instance_initiated_shutdown_behavior = try(each.value.instance_initiated_shutdown_behavior, var.self_managed_node_group_defaults.instance_initiated_shutdown_behavior, null) + kernel_id = try(each.value.kernel_id, var.self_managed_node_group_defaults.kernel_id, null) + ram_disk_id = try(each.value.ram_disk_id, var.self_managed_node_group_defaults.ram_disk_id, null) block_device_mappings = try(each.value.block_device_mappings, var.self_managed_node_group_defaults.block_device_mappings, {}) capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.self_managed_node_group_defaults.capacity_reservation_specification, {}) @@ -429,12 +432,15 @@ module "self_managed_node_group" { elastic_inference_accelerator = try(each.value.elastic_inference_accelerator, var.self_managed_node_group_defaults.elastic_inference_accelerator, {}) enclave_options = try(each.value.enclave_options, var.self_managed_node_group_defaults.enclave_options, {}) hibernation_options = try(each.value.hibernation_options, var.self_managed_node_group_defaults.hibernation_options, {}) + instance_requirements = try(each.value.instance_requirements, var.self_managed_node_group_defaults.instance_requirements, {}) instance_market_options = try(each.value.instance_market_options, var.self_managed_node_group_defaults.instance_market_options, {}) license_specifications = try(each.value.license_specifications, var.self_managed_node_group_defaults.license_specifications, {}) metadata_options = try(each.value.metadata_options, var.self_managed_node_group_defaults.metadata_options, local.metadata_options) enable_monitoring = try(each.value.enable_monitoring, var.self_managed_node_group_defaults.enable_monitoring, true) network_interfaces = try(each.value.network_interfaces, var.self_managed_node_group_defaults.network_interfaces, []) placement = try(each.value.placement, var.self_managed_node_group_defaults.placement, {}) + maintenance_options = try(each.value.maintenance_options, var.self_managed_node_group_defaults.maintenance_options, {}) + private_dns_name_options = try(each.value.private_dns_name_options, var.self_managed_node_group_defaults.private_dns_name_options, {}) # IAM role create_iam_instance_profile = try(each.value.create_iam_instance_profile, var.self_managed_node_group_defaults.create_iam_instance_profile, true) @@ -446,19 +452,13 @@ module "self_managed_node_group" { iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.self_managed_node_group_defaults.iam_role_permissions_boundary, null) iam_role_tags = try(each.value.iam_role_tags, var.self_managed_node_group_defaults.iam_role_tags, {}) iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.self_managed_node_group_defaults.iam_role_attach_cni_policy, true) - iam_role_additional_policies = try(each.value.iam_role_additional_policies, var.self_managed_node_group_defaults.iam_role_additional_policies, []) + # To better understand why this `lookup()` logic is required, see: + # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 + iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.self_managed_node_group_defaults, "iam_role_additional_policies", {})) # Security group vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.self_managed_node_group_defaults.vpc_security_group_ids, []))) - cluster_security_group_id = local.cluster_security_group_id cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, var.self_managed_node_group_defaults.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null - create_security_group = try(each.value.create_security_group, var.self_managed_node_group_defaults.create_security_group, true) - security_group_name = try(each.value.security_group_name, var.self_managed_node_group_defaults.security_group_name, null) - security_group_use_name_prefix = try(each.value.security_group_use_name_prefix, var.self_managed_node_group_defaults.security_group_use_name_prefix, true) - security_group_description = try(each.value.security_group_description, var.self_managed_node_group_defaults.security_group_description, "Self managed node group security group") - vpc_id = try(each.value.vpc_id, var.self_managed_node_group_defaults.vpc_id, var.vpc_id) - security_group_rules = try(each.value.security_group_rules, var.self_managed_node_group_defaults.security_group_rules, {}) - security_group_tags = try(each.value.security_group_tags, var.self_managed_node_group_defaults.security_group_tags, {}) tags = merge(var.tags, try(each.value.tags, var.self_managed_node_group_defaults.tags, {})) } diff --git a/outputs.tf b/outputs.tf index eadf2d0cd2..dec0540174 100644 --- a/outputs.tf +++ b/outputs.tf @@ -4,52 +4,52 @@ output "cluster_arn" { description = "The Amazon Resource Name (ARN) of the cluster" - value = try(aws_eks_cluster.this[0].arn, "") + value = try(aws_eks_cluster.this[0].arn, null) } output "cluster_certificate_authority_data" { description = "Base64 encoded certificate data required to communicate with the cluster" - value = try(aws_eks_cluster.this[0].certificate_authority[0].data, "") + value = try(aws_eks_cluster.this[0].certificate_authority[0].data, null) } output "cluster_endpoint" { description = "Endpoint for your Kubernetes API server" - value = try(aws_eks_cluster.this[0].endpoint, "") + value = try(aws_eks_cluster.this[0].endpoint, null) } -output "cluster_name" { - description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" - value = try(aws_eks_cluster.this[0].name, "") +output "cluster_id" { + description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" + value = try(aws_eks_cluster.this[0].cluster_id, null) } -output "cluster_id" { - description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" - value = try(aws_eks_cluster.this[0].id, "") +output "cluster_name" { + description = "The name of the EKS cluster" + value = try(aws_eks_cluster.this[0].name, null) } output "cluster_oidc_issuer_url" { description = "The URL on the EKS cluster for the OpenID Connect identity provider" - value = try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "") + value = try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, null) } output "cluster_version" { description = "The Kubernetes version for the cluster" - value = try(aws_eks_cluster.this[0].version, "") + value = try(aws_eks_cluster.this[0].version, null) } output "cluster_platform_version" { description = "Platform version for the cluster" - value = try(aws_eks_cluster.this[0].platform_version, "") + value = try(aws_eks_cluster.this[0].platform_version, null) } output "cluster_status" { description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" - value = try(aws_eks_cluster.this[0].status, "") + value = try(aws_eks_cluster.this[0].status, null) } output "cluster_primary_security_group_id" { description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" - value = try(aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id, "") + value = try(aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id, null) } ################################################################################ @@ -77,12 +77,12 @@ output "kms_key_policy" { output "cluster_security_group_arn" { description = "Amazon Resource Name (ARN) of the cluster security group" - value = try(aws_security_group.cluster[0].arn, "") + value = try(aws_security_group.cluster[0].arn, null) } output "cluster_security_group_id" { description = "ID of the cluster security group" - value = try(aws_security_group.cluster[0].id, "") + value = try(aws_security_group.cluster[0].id, null) } ################################################################################ @@ -91,12 +91,12 @@ output "cluster_security_group_id" { output "node_security_group_arn" { description = "Amazon Resource Name (ARN) of the node shared security group" - value = try(aws_security_group.node[0].arn, "") + value = try(aws_security_group.node[0].arn, null) } output "node_security_group_id" { description = "ID of the node shared security group" - value = try(aws_security_group.node[0].id, "") + value = try(aws_security_group.node[0].id, null) } ################################################################################ @@ -105,17 +105,17 @@ output "node_security_group_id" { output "oidc_provider" { description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" - value = try(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://", ""), "") + value = try(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://", null), null) } output "oidc_provider_arn" { description = "The ARN of the OIDC Provider if `enable_irsa = true`" - value = try(aws_iam_openid_connect_provider.oidc_provider[0].arn, "") + value = try(aws_iam_openid_connect_provider.oidc_provider[0].arn, null) } output "cluster_tls_certificate_sha1_fingerprint" { description = "The SHA1 fingerprint of the public key of the cluster's certificate" - value = try(data.tls_certificate.this[0].certificates[0].sha1_fingerprint, "") + value = try(data.tls_certificate.this[0].certificates[0].sha1_fingerprint, null) } ################################################################################ @@ -124,17 +124,17 @@ output "cluster_tls_certificate_sha1_fingerprint" { output "cluster_iam_role_name" { description = "IAM role name of the EKS cluster" - value = try(aws_iam_role.this[0].name, "") + value = try(aws_iam_role.this[0].name, null) } output "cluster_iam_role_arn" { description = "IAM role ARN of the EKS cluster" - value = try(aws_iam_role.this[0].arn, "") + value = try(aws_iam_role.this[0].arn, null) } output "cluster_iam_role_unique_id" { description = "Stable and unique string identifying the IAM role" - value = try(aws_iam_role.this[0].unique_id, "") + value = try(aws_iam_role.this[0].unique_id, null) } ################################################################################ @@ -161,12 +161,12 @@ output "cluster_identity_providers" { output "cloudwatch_log_group_name" { description = "Name of cloudwatch log group created" - value = try(aws_cloudwatch_log_group.this[0].name, "") + value = try(aws_cloudwatch_log_group.this[0].name, null) } output "cloudwatch_log_group_arn" { description = "Arn of cloudwatch log group created" - value = try(aws_cloudwatch_log_group.this[0].arn, "") + value = try(aws_cloudwatch_log_group.this[0].arn, null) } ################################################################################ diff --git a/variables.tf b/variables.tf index 6bce0560e2..d0047ff1c0 100644 --- a/variables.tf +++ b/variables.tf @@ -59,13 +59,13 @@ variable "subnet_ids" { variable "cluster_endpoint_private_access" { description = "Indicates whether or not the Amazon EKS private API server endpoint is enabled" type = bool - default = false + default = true } variable "cluster_endpoint_public_access" { description = "Indicates whether or not the Amazon EKS public API server endpoint is enabled" type = bool - default = true + default = false } variable "cluster_endpoint_public_access_cidrs" { @@ -86,10 +86,24 @@ variable "cluster_service_ipv4_cidr" { default = null } +variable "cluster_service_ipv6_cidr" { + description = "The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster" + type = string + default = null +} + +variable "outpost_config" { + description = "Configuration for the AWS Outpost to provision the cluster on" + type = any + default = {} +} + variable "cluster_encryption_config" { description = "Configuration block with encryption configuration for the cluster" - type = list(any) - default = [] + type = any + default = { + resources = ["secrets"] + } } variable "attach_cluster_encryption_policy" { @@ -123,7 +137,7 @@ variable "cluster_timeouts" { variable "create_kms_key" { description = "Controls if a KMS key for cluster encryption should be created" type = bool - default = false + default = true } variable "kms_key_description" { @@ -219,19 +233,19 @@ variable "cloudwatch_log_group_kms_key_id" { ################################################################################ variable "create_cluster_security_group" { - description = "Determines if a security group is created for the cluster or use the existing `cluster_security_group_id`" + description = "Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default" type = bool default = true } variable "cluster_security_group_id" { - description = "Existing security group ID to be attached to the cluster. Required if `create_cluster_security_group` = `false`" + description = "Existing security group ID to be attached to the cluster" type = string default = "" } variable "vpc_id" { - description = "ID of the VPC where the cluster and its nodes will be provisioned" + description = "ID of the VPC where the cluster security group will be provisioned" type = string default = null } @@ -316,26 +330,18 @@ variable "node_security_group_additional_rules" { default = {} } +variable "node_security_group_enable_recommended_rules" { + description = "Determines whether to enable recommended security group rules for the node security group created. This includes node-to-node TCP ingress on ephemeral ports and allows all egress traffic" + type = bool + default = true +} + variable "node_security_group_tags" { description = "A map of additional tags to add to the node security group created" type = map(string) default = {} } -# TODO - at next breaking change, make 169.254.169.123/32 the default -variable "node_security_group_ntp_ipv4_cidr_block" { - description = "IPv4 CIDR block to allow NTP egress. Default is public IP space, but [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) can be used as well with `[\"169.254.169.123/32\"]`" - type = list(string) - default = ["0.0.0.0/0"] -} - -# TODO - at next breaking change, make fd00:ec2::123/128 the default -variable "node_security_group_ntp_ipv6_cidr_block" { - description = "IPv4 CIDR block to allow NTP egress. Default is public IP space, but [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) can be used as well with `[\"fd00:ec2::123/128\"]`" - type = list(string) - default = ["::/0"] -} - ################################################################################ # IRSA ################################################################################ @@ -406,8 +412,8 @@ variable "iam_role_permissions_boundary" { variable "iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" - type = list(string) - default = [] + type = map(string) + default = {} } # TODO - hopefully this can be removed once the AWS endpoint is named properly in China @@ -464,6 +470,12 @@ variable "cluster_addons" { default = {} } +variable "cluster_addons_timeouts" { + description = "Create, update, and delete timeout configurations for the cluster addons" + type = map(string) + default = {} +} + ################################################################################ # EKS Identity Provider ################################################################################ diff --git a/versions.tf b/versions.tf index fde7af0f23..fdc407c2ed 100644 --- a/versions.tf +++ b/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72" + version = ">= 4.45" } tls = { source = "hashicorp/tls" From 4bb83e0165dc85d75899b6c107b2e42bb5157b55 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 5 Dec 2022 21:26:51 +0000 Subject: [PATCH 609/962] chore(release): version 19.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [19.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.31.2...v19.0.0) (2022-12-05) ### ⚠ BREAKING CHANGES * Add support for Outposts, remove node security group, add support for addon `preserve` and `most_recent` configurations (#2250) ### Features * Add support for Outposts, remove node security group, add support for addon `preserve` and `most_recent` configurations ([#2250](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2250)) ([b2e97ca](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b2e97ca3dcbcd76063f1c932aa5199b4f49a2aa1)) --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c48e32e975..82ec9fd89f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. +## [19.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.31.2...v19.0.0) (2022-12-05) + + +### ⚠ BREAKING CHANGES + +* Add support for Outposts, remove node security group, add support for addon `preserve` and `most_recent` configurations (#2250) + +### Features + +* Add support for Outposts, remove node security group, add support for addon `preserve` and `most_recent` configurations ([#2250](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2250)) ([b2e97ca](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b2e97ca3dcbcd76063f1c932aa5199b4f49a2aa1)) + ### [18.31.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.31.1...v18.31.2) (2022-11-23) From 7bc4a2743f0cdf9c8556a2c067eeb82436aafb41 Mon Sep 17 00:00:00 2001 From: Adam Staudt <54176201+AdamStaudt@users.noreply.github.com> Date: Tue, 6 Dec 2022 20:10:29 +0100 Subject: [PATCH 610/962] fix: Call to lookup() closed too early, breaks sg rule creation in cluster sg if custom source sg is defined. (#2319) Co-authored-by: Bryant Biggs --- examples/complete/main.tf | 18 ++++++++++++++++++ main.tf | 13 ++++++------- node_groups.tf | 13 ++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index a040d750f2..c16704b499 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -84,6 +84,15 @@ module "eks" { type = "ingress" source_node_security_group = true } + # Test: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2319 + ingress_source_security_group_id = { + description = "Ingress from another computed security group" + protocol = "tcp" + from_port = 22 + to_port = 22 + type = "ingress" + source_security_group_id = aws_security_group.additional.id + } } # Extend node-to-node security group rules @@ -96,6 +105,15 @@ module "eks" { type = "ingress" self = true } + # Test: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2319 + ingress_source_security_group_id = { + description = "Ingress from another computed security group" + protocol = "tcp" + from_port = 22 + to_port = 22 + type = "ingress" + source_security_group_id = aws_security_group.additional.id + } } # Self Managed Node Group(s) diff --git a/main.tf b/main.tf index 9193031348..ae5a0366e5 100644 --- a/main.tf +++ b/main.tf @@ -191,13 +191,12 @@ resource "aws_security_group_rule" "cluster" { type = each.value.type # Optional - description = lookup(each.value, "description", null) - cidr_blocks = lookup(each.value, "cidr_blocks", null) - ipv6_cidr_blocks = lookup(each.value, "ipv6_cidr_blocks", null) - prefix_list_ids = lookup(each.value, "prefix_list_ids", []) - self = lookup(each.value, "self", null) - source_security_group_id = lookup(each.value, "source_security_group_id", - lookup(each.value, "source_node_security_group", false)) ? local.node_security_group_id : null + description = lookup(each.value, "description", null) + cidr_blocks = lookup(each.value, "cidr_blocks", null) + ipv6_cidr_blocks = lookup(each.value, "ipv6_cidr_blocks", null) + prefix_list_ids = lookup(each.value, "prefix_list_ids", null) + self = lookup(each.value, "self", null) + source_security_group_id = try(each.value.source_node_security_group, false) ? local.node_security_group_id : lookup(each.value, "source_security_group_id", null) } ################################################################################ diff --git a/node_groups.tf b/node_groups.tf index 36f071610c..07ec3ecda7 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -180,13 +180,12 @@ resource "aws_security_group_rule" "node" { type = each.value.type # Optional - description = lookup(each.value, "description", null) - cidr_blocks = lookup(each.value, "cidr_blocks", null) - ipv6_cidr_blocks = lookup(each.value, "ipv6_cidr_blocks", null) - prefix_list_ids = lookup(each.value, "prefix_list_ids", []) - self = lookup(each.value, "self", null) - source_security_group_id = lookup(each.value, "source_security_group_id", - lookup(each.value, "source_cluster_security_group", false)) ? local.cluster_security_group_id : null + description = lookup(each.value, "description", null) + cidr_blocks = lookup(each.value, "cidr_blocks", null) + ipv6_cidr_blocks = lookup(each.value, "ipv6_cidr_blocks", null) + prefix_list_ids = lookup(each.value, "prefix_list_ids", []) + self = lookup(each.value, "self", null) + source_security_group_id = try(each.value.source_cluster_security_group, false) ? local.cluster_security_group_id : lookup(each.value, "source_security_group_id", null) } ################################################################################ From 2def94ed2241e355df02605c4e1777e90751c295 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 6 Dec 2022 19:11:00 +0000 Subject: [PATCH 611/962] chore(release): version 19.0.1 [skip ci] ### [19.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.0...v19.0.1) (2022-12-06) ### Bug Fixes * Call to lookup() closed too early, breaks sg rule creation in cluster sg if custom source sg is defined. ([#2319](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2319)) ([7bc4a27](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7bc4a2743f0cdf9c8556a2c067eeb82436aafb41)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82ec9fd89f..f3ecc5fbd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.0...v19.0.1) (2022-12-06) + + +### Bug Fixes + +* Call to lookup() closed too early, breaks sg rule creation in cluster sg if custom source sg is defined. ([#2319](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2319)) ([7bc4a27](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7bc4a2743f0cdf9c8556a2c067eeb82436aafb41)) + ## [19.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.31.2...v19.0.0) (2022-12-05) From 3f6d915eef6672440df8c82468c31ed2bc2fce54 Mon Sep 17 00:00:00 2001 From: Sebastian Melchior <801781+smelchior@users.noreply.github.com> Date: Tue, 6 Dec 2022 22:55:02 +0100 Subject: [PATCH 612/962] fix: `public_access_cidrs` require a value even if public endpoint is disabled (#2320) Co-authored-by: Bryant Biggs --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index ae5a0366e5..15e777d03c 100644 --- a/main.tf +++ b/main.tf @@ -27,7 +27,7 @@ resource "aws_eks_cluster" "this" { subnet_ids = coalescelist(var.control_plane_subnet_ids, var.subnet_ids) endpoint_private_access = var.cluster_endpoint_private_access endpoint_public_access = var.cluster_endpoint_public_access - public_access_cidrs = var.cluster_endpoint_public_access ? var.cluster_endpoint_public_access_cidrs : [] + public_access_cidrs = var.cluster_endpoint_public_access_cidrs } dynamic "kubernetes_network_config" { From f09a6432bb9d7f74ede3717da73e3360ad903605 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 6 Dec 2022 21:55:38 +0000 Subject: [PATCH 613/962] chore(release): version 19.0.2 [skip ci] ### [19.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.1...v19.0.2) (2022-12-06) ### Bug Fixes * `public_access_cidrs` require a value even if public endpoint is disabled ([#2320](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2320)) ([3f6d915](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3f6d915eef6672440df8c82468c31ed2bc2fce54)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3ecc5fbd5..cac0db9bdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.1...v19.0.2) (2022-12-06) + + +### Bug Fixes + +* `public_access_cidrs` require a value even if public endpoint is disabled ([#2320](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2320)) ([3f6d915](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3f6d915eef6672440df8c82468c31ed2bc2fce54)) + ### [19.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.0...v19.0.1) (2022-12-06) From 9adc475bc1f1a201648e37b26cefe9bdf6b3a2f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Pozo?= Date: Tue, 6 Dec 2022 22:26:12 -0300 Subject: [PATCH 614/962] fix: Invalid value for "replace" parameter: argument must not be null. (#2322) --- outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/outputs.tf b/outputs.tf index dec0540174..a0795ec736 100644 --- a/outputs.tf +++ b/outputs.tf @@ -105,7 +105,7 @@ output "node_security_group_id" { output "oidc_provider" { description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" - value = try(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://", null), null) + value = try(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://", ""), null) } output "oidc_provider_arn" { From c0423efb94d1536845479261f8eddb595232a765 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 7 Dec 2022 01:26:46 +0000 Subject: [PATCH 615/962] chore(release): version 19.0.3 [skip ci] ### [19.0.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.2...v19.0.3) (2022-12-07) ### Bug Fixes * Invalid value for "replace" parameter: argument must not be null. ([#2322](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2322)) ([9adc475](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/9adc475bc1f1a201648e37b26cefe9bdf6b3a2f7)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cac0db9bdc..7ef90c05ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.0.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.2...v19.0.3) (2022-12-07) + + +### Bug Fixes + +* Invalid value for "replace" parameter: argument must not be null. ([#2322](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2322)) ([9adc475](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/9adc475bc1f1a201648e37b26cefe9bdf6b3a2f7)) + ### [19.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.1...v19.0.2) (2022-12-06) From b83f6d98bfbca548012ea74e792fe14f04f0e6dc Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Wed, 7 Dec 2022 11:05:49 -0500 Subject: [PATCH 616/962] fix: Ensure that custom KMS key is not created if encryption is not enabled, support computed values in cluster name (#2328) Co-authored-by: Bryant Biggs Resolves undefined Resolved undefined Closes undefined --- examples/complete/README.md | 2 ++ examples/complete/main.tf | 22 +++++++++++++++++----- main.tf | 8 ++++++-- modules/eks-managed-node-group/main.tf | 2 +- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/examples/complete/README.md b/examples/complete/README.md index 6f5edbdb41..f6bac2e304 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -54,6 +54,7 @@ Note that this example may create resources which cost money. Run `terraform des | [eks](#module\_eks) | ../.. | n/a | | [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | | [fargate\_profile](#module\_fargate\_profile) | ../../modules/fargate-profile | n/a | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 1.1.0 | | [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ../../modules/self-managed-node-group | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | @@ -64,6 +65,7 @@ Note that this example may create resources which cost money. Run `terraform des | [aws_iam_policy.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | ## Inputs diff --git a/examples/complete/main.tf b/examples/complete/main.tf index c16704b499..2917c5bc37 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -15,6 +15,7 @@ provider "kubernetes" { } data "aws_availability_zones" "available" {} +data "aws_caller_identity" "current" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" @@ -58,13 +59,12 @@ module "eks" { } } - # Encryption key - create_kms_key = true + # External encryption key + create_kms_key = false cluster_encryption_config = { - resources = ["secrets"] + resources = ["secrets"] + provider_key_arn = module.kms.key_arn } - kms_key_deletion_window_in_days = 7 - enable_kms_key_rotation = true iam_role_additional_policies = { additional = aws_iam_policy.additional.arn @@ -460,3 +460,15 @@ resource "aws_iam_policy" "additional" { ] }) } + +module "kms" { + source = "terraform-aws-modules/kms/aws" + version = "1.1.0" + + aliases = ["eks/${local.name}"] + description = "${local.name} cluster encryption key" + enable_default_policy = true + key_owners = [data.aws_caller_identity.current.arn] + + tags = local.tags +} diff --git a/main.tf b/main.tf index 15e777d03c..3d09417aaf 100644 --- a/main.tf +++ b/main.tf @@ -112,7 +112,7 @@ module "kms" { source = "terraform-aws-modules/kms/aws" version = "1.1.0" # Note - be mindful of Terraform/provider version compatibility between modules - create = local.create && var.create_kms_key && !local.create_outposts_local_cluster # not valid on Outposts + create = local.create && var.create_kms_key && local.enable_cluster_encryption_config # not valid on Outposts description = coalesce(var.kms_key_description, "${var.cluster_name} cluster encryption key") key_usage = "ENCRYPT_DECRYPT" @@ -129,7 +129,11 @@ module "kms" { override_policy_documents = var.kms_key_override_policy_documents # Aliases - aliases = concat(["eks/${var.cluster_name}"], var.kms_key_aliases) + aliases = var.kms_key_aliases + computed_aliases = { + # Computed since users can pass in computed values for cluster name such as random provider resources + cluster = { name = "eks/${var.cluster_name}" } + } tags = var.tags } diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 06275f00a2..b844533abe 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -300,7 +300,7 @@ resource "aws_launch_template" "this" { ################################################################################ locals { - launch_template_id = var.create && var.create_launch_template ? aws_launch_template.this[0].id : var.launch_template_id + launch_template_id = var.create && var.create_launch_template ? try(aws_launch_template.this[0].id, null) : var.launch_template_id # Change order to allow users to set version priority before using defaults launch_template_version = coalesce(var.launch_template_version, try(aws_launch_template.this[0].default_version, "$Default")) } From 7124d7603a81ad07bbcc7a932ba0f41989365471 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 7 Dec 2022 16:06:21 +0000 Subject: [PATCH 617/962] chore(release): version 19.0.4 [skip ci] ### [19.0.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.3...v19.0.4) (2022-12-07) ### Bug Fixes * Ensure that custom KMS key is not created if encryption is not enabled, support computed values in cluster name ([#2328](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2328)) ([b83f6d9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b83f6d98bfbca548012ea74e792fe14f04f0e6dc)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ef90c05ef..7c3a58c3b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.0.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.3...v19.0.4) (2022-12-07) + + +### Bug Fixes + +* Ensure that custom KMS key is not created if encryption is not enabled, support computed values in cluster name ([#2328](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2328)) ([b83f6d9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b83f6d98bfbca548012ea74e792fe14f04f0e6dc)) + ### [19.0.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.2...v19.0.3) (2022-12-07) From ca03fd9ec16f8e6d8440d23adb72cb7c7ba33c56 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 8 Dec 2022 09:35:28 -0500 Subject: [PATCH 618/962] docs: Correct spelling mistakes (#2334) Resolves undefined --- README.md | 2 +- docs/UPGRADE-19.0.md | 12 ++++++------ examples/eks_managed_node_group/README.md | 2 +- examples/eks_managed_node_group/main.tf | 4 ++-- examples/fargate_profile/main.tf | 6 +++--- examples/karpenter/main.tf | 6 +++--- main.tf | 2 +- modules/_user_data/main.tf | 2 +- node_groups.tf | 6 +++--- 9 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index fb7dd4845f..a0d1fce5c2 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ The examples provided under `examples/` provide a comprehensive suite of configu - [EKS Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) - [Self Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) - [Fargate Profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) -- Support for creating Karpenter related AWS infrastruture resources (e.g. IAM roles, SQS queue, EventBridge rules, etc.) +- Support for creating Karpenter related AWS infrastructure resources (e.g. IAM roles, SQS queue, EventBridge rules, etc.) - Support for custom AMI, custom launch template, and custom user data including custom user data template - Support for Amazon Linux 2 EKS Optimized AMI and Bottlerocket nodes - Windows based node support is limited to a default user data template that is provided due to the lack of Windows support and manual steps required to provision Windows based EKS nodes diff --git a/docs/UPGRADE-19.0.md b/docs/UPGRADE-19.0.md index 429a6a8cbd..4bef119746 100644 --- a/docs/UPGRADE-19.0.md +++ b/docs/UPGRADE-19.0.md @@ -8,7 +8,7 @@ Please consult the `examples` directory for reference example configurations. If - Minimum supported version of Terraform AWS provider updated to v4.45 to support latest features provided via the resources utilized. - Minimum supported version of Terraform updated to v1.0 - Individual security group created per EKS managed node group or self managed node group has been removed. This configuration went mostly un-used and would often cause confusion ("Why is there an empty security group attached to my nodes?"). This functionality can easily be replicated by user's providing one or more externally created security groups to attach to nodes launched from the node group. -- Previously, `var.iam_role_additional_policies` (one for each of the following: cluster IAM role, EKS managed node group IAM role, self-managed node group IAM role, and Fargate Profile IAM role) accepted a list of strings. This worked well for policies that already existed but failed for policies being created at the same time as the cluster due to the well known issue of unkown values used in a `for_each` loop. To rectify this issue in `v19.x`, two changes were made: +- Previously, `var.iam_role_additional_policies` (one for each of the following: cluster IAM role, EKS managed node group IAM role, self-managed node group IAM role, and Fargate Profile IAM role) accepted a list of strings. This worked well for policies that already existed but failed for policies being created at the same time as the cluster due to the well known issue of unknown values used in a `for_each` loop. To rectify this issue in `v19.x`, two changes were made: 1. `var.iam_role_additional_policies` was changed from type `list(string)` to type `map(string)` -> this is a breaking change. More information on managing this change can be found below, under `Terraform State Moves` 2. The logic used in the root module for this variable was changed to replace the use of `try()` with `lookup()`. More details on why can be found [here](https://github.com/clowdhaus/terraform-for-each-unknown) - The cluster name has been removed from the Karpenter module event rule names. Due to the use of long cluster names appending to the provided naming scheme, the cluster name has moved to a `ClusterName` tag and the event rule name is now a prefix. This guarantees that users can have multiple instances of Karpenter withe their respective event rules/SQS queue without name collisions, while also still being able to identify which queues and event rules belong to which cluster. @@ -26,7 +26,7 @@ Please consult the `examples` directory for reference example configurations. If ### Modified -- `cluster_security_group_additional_rules` and `node_security_group_additional_rules` have been modified to use `lookup()` instead of `try()` to avoid the well known issue of [unkown values within a `for_each` loop](https://github.com/hashicorp/terraform/issues/4149) +- `cluster_security_group_additional_rules` and `node_security_group_additional_rules` have been modified to use `lookup()` instead of `try()` to avoid the well known issue of [unknown values within a `for_each` loop](https://github.com/hashicorp/terraform/issues/4149) - Default cluster security group rules have removed egress rules for TCP/443 and TCP/10250 to node groups since the cluster primary security group includes a default rule for ALL to `0.0.0.0/0`/`::/0` - Default node security group rules have removed egress rules have been removed since the default security group settings have egress rule for ALL to `0.0.0.0/0`/`::/0` - `block_device_mappings` previously required a map of maps but has since changed to an array of maps. Users can remove the outer key for each block device mapping and replace the outermost map `{}` with an array `[]`. There are no state changes required for this change. @@ -101,7 +101,7 @@ Please consult the `examples` directory for reference example configurations. If - `default_instance_warmup` - `force_delete_warm_pool` - EKS managed node groups: - - `use_custom_launch_template` was added to better clarify how users can switch betweeen a custom launch template or the default launch template provided by the EKS managed node group. Previously, to achieve this same functionality of using the default launch template, users needed to set `create_launch_template = false` and `launch_template_name = ""` which is not very intuitive. + - `use_custom_launch_template` was added to better clarify how users can switch between a custom launch template or the default launch template provided by the EKS managed node group. Previously, to achieve this same functionality of using the default launch template, users needed to set `create_launch_template = false` and `launch_template_name = ""` which is not very intuitive. - `launch_template_id` for use when using an existing/externally created launch template (Ref: https://github.com/terraform-aws-modules/terraform-aws-autoscaling/pull/204) - `maintenance_options` - `private_dns_name_options` @@ -121,7 +121,7 @@ Please consult the `examples` directory for reference example configurations. If 6. Added outputs: - - `cluster_name` - The `cluster_id` currently set by the AWS provider is actually the cluster name, but in the future this will change and there will be a distinction between the `cluster_name` and `clsuter_id`. [Reference](https://github.com/hashicorp/terraform-provider-aws/issues/27560) + - `cluster_name` - The `cluster_id` currently set by the AWS provider is actually the cluster name, but in the future this will change and there will be a distinction between the `cluster_name` and `cluster_id`. [Reference](https://github.com/hashicorp/terraform-provider-aws/issues/27560) ## Upgrade Migrations @@ -165,7 +165,7 @@ EKS managed node groups on `v18.x` by default create a security group that does 2. Once the node group security group(s) have been removed, you can update your module definition to specify the `v19.x` version of the module 3. Run `terraform init -upgrade=true` to update your configuration and pull in the v19 changes 4. Using the documentation provided above, update your module definition to reflect the changes in the module from `v18.x` to `v19.x`. You can utilize `terraform plan` as you go to help highlight any changes that you wish to make. See below for `terraform state mv ...` commands related to the use of `iam_role_additional_policies`. If you are not providing any values to these variables, you can skip this section. -5. Once you are satisifed with the changes and the `terraform plan` output, you can apply the changes to sync your infrastructure with the updated module definition (or vice versa). +5. Once you are satisfied with the changes and the `terraform plan` output, you can apply the changes to sync your infrastructure with the updated module definition (or vice versa). ### Diff of Before (v18.x) vs After (v19.x) @@ -418,7 +418,7 @@ Where `""` is specified, this should be replaced with the full ARN o ```hcl ... - # This is demonstrating the cluster IAM role addtional policies + # This is demonstrating the cluster IAM role additional policies iam_role_additional_policies = { additional = aws_iam_policy.additional.arn } diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 42fb41e82c..ecb37f380c 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -14,7 +14,7 @@ See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/man ## Container Runtime & User Data -When using the default AMI provided by the EKS Managed Node Group service (i.e. - not specifying a value for `ami_id`), users should be aware of the limitations of configuring the node bootstrap process via user data. Due to not having direct access to the bootrap.sh script invocation and therefore its configuration flags (this is provided by the EKS Managed Node Group service in the node user data), a workaround for ensuring the appropriate configuration settings is shown below. The following example shows how to inject configuration variables ahead of the merged user data provided by the EKS Managed Node Group service as well as how to enable the containerd runtime using this approach. More details can be found [here](https://github.com/awslabs/amazon-eks-ami/issues/844). +When using the default AMI provided by the EKS Managed Node Group service (i.e. - not specifying a value for `ami_id`), users should be aware of the limitations of configuring the node bootstrap process via user data. Due to not having direct access to the bootstrap.sh script invocation and therefore its configuration flags (this is provided by the EKS Managed Node Group service in the node user data), a workaround for ensuring the appropriate configuration settings is shown below. The following example shows how to inject configuration variables ahead of the merged user data provided by the EKS Managed Node Group service as well as how to enable the containerd runtime using this approach. More details can be found [here](https://github.com/awslabs/amazon-eks-ami/issues/844). ```hcl ... diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index ebf39cdd16..378496c7f2 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -130,7 +130,7 @@ module "eks" { ami_id = data.aws_ami.eks_default_bottlerocket.image_id platform = "bottlerocket" - # Use module user data template to boostrap + # Use module user data template to bootstrap enable_bootstrap_user_data = true # This will get added to the template bootstrap_extra_args = <<-EOT @@ -165,7 +165,7 @@ module "eks" { # Current default AMI used by managed node groups - pseudo "custom" ami_id = data.aws_ami.eks_default_arm.image_id - # This will ensure the boostrap user data is used to join the node + # This will ensure the bootstrap user data is used to join the node # By default, EKS managed node groups will not append bootstrap script; # this adds it back in using the default template provided by the module # Note: this assumes the AMI provided is an EKS optimized AMI derivative diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index 8eb0c934fe..fa6c8bdb64 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -152,7 +152,7 @@ resource "null_resource" "remove_default_coredns_deployment" { } # We are removing the deployment provided by the EKS service and replacing it through the self-managed CoreDNS Helm addon - # However, we are maintaing the existing kube-dns service and annotating it for Helm to assume control + # However, we are maintaining the existing kube-dns service and annotating it for Helm to assume control command = <<-EOT kubectl --namespace kube-system delete deployment coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) EOT @@ -168,7 +168,7 @@ resource "null_resource" "modify_kube_dns" { KUBECONFIG = base64encode(local.kubeconfig) } - # We are maintaing the existing kube-dns service and annotating it for Helm to assume control + # We are maintaining the existing kube-dns service and annotating it for Helm to assume control command = <<-EOT echo "Setting implicit dependency on ${module.eks.fargate_profiles["kube_system"].fargate_profile_pod_execution_role_arn}" kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-name=coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) @@ -223,7 +223,7 @@ resource "helm_release" "coredns" { ] depends_on = [ - # Need to ensure the CoreDNS updates are peformed before provisioning + # Need to ensure the CoreDNS updates are performed before provisioning null_resource.modify_kube_dns ] } diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 0a8725e466..eeb44ac0fc 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -292,7 +292,7 @@ resource "null_resource" "remove_default_coredns_deployment" { } # We are removing the deployment provided by the EKS service and replacing it through the self-managed CoreDNS Helm addon - # However, we are maintaing the existing kube-dns service and annotating it for Helm to assume control + # However, we are maintaining the existing kube-dns service and annotating it for Helm to assume control command = <<-EOT kubectl --namespace kube-system delete deployment coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) EOT @@ -308,7 +308,7 @@ resource "null_resource" "modify_kube_dns" { KUBECONFIG = base64encode(local.kubeconfig) } - # We are maintaing the existing kube-dns service and annotating it for Helm to assume control + # We are maintaining the existing kube-dns service and annotating it for Helm to assume control command = <<-EOT echo "Setting implicit dependency on ${module.eks.fargate_profiles["kube_system"].fargate_profile_pod_execution_role_arn}" kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-name=coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) @@ -363,7 +363,7 @@ resource "helm_release" "coredns" { ] depends_on = [ - # Need to ensure the CoreDNS updates are peformed before provisioning + # Need to ensure the CoreDNS updates are performed before provisioning null_resource.modify_kube_dns ] } diff --git a/main.tf b/main.tf index 3d09417aaf..0001d55f6c 100644 --- a/main.tf +++ b/main.tf @@ -283,7 +283,7 @@ resource "aws_iam_role" "this" { force_detach_policies = true # https://github.com/terraform-aws-modules/terraform-aws-eks/issues/920 - # Resources running on the cluster are still generaring logs when destroying the module resources + # Resources running on the cluster are still generating logs when destroying the module resources # which results in the log group being re-created even after Terraform destroys it. Removing the # ability for the cluster role to create the log group prevents this log group from being re-created # outside of Terraform due to services still generating logs during destroy process diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index 8eec42d36c..8ace10539d 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -70,7 +70,7 @@ data "cloudinit_config" "linux_eks_managed_node_group" { gzip = false boundary = "//" - # Prepend to existing user data suppled by AWS EKS + # Prepend to existing user data supplied by AWS EKS part { content_type = "text/x-shellscript" content = var.pre_bootstrap_user_data diff --git a/node_groups.tf b/node_groups.tf index 07ec3ecda7..6561faec5a 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -47,7 +47,7 @@ data "aws_iam_policy_document" "cni_ipv6_policy" { } } -# Note - we are keeping this to a minimim in hopes that its soon replaced with an AWS managed policy like `AmazonEKS_CNI_Policy` +# Note - we are keeping this to a minimum in hopes that its soon replaced with an AWS managed policy like `AmazonEKS_CNI_Policy` resource "aws_iam_policy" "cni_ipv6_policy" { count = var.create && var.create_cni_ipv6_iam_policy ? 1 : 0 @@ -106,7 +106,7 @@ locals { } } - node_secuirty_group_recommended_rules = { for k, v in { + node_security_group_recommended_rules = { for k, v in { ingress_nodes_ephemeral = { description = "Node to node ingress on ephemeral ports" protocol = "tcp" @@ -168,7 +168,7 @@ resource "aws_security_group" "node" { resource "aws_security_group_rule" "node" { for_each = { for k, v in merge( local.node_security_group_rules, - local.node_secuirty_group_recommended_rules, + local.node_security_group_recommended_rules, var.node_security_group_additional_rules, ) : k => v if local.create_node_sg } From 3b62f6c31604490fc19184e626e73873b296ecd1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 16 Dec 2022 14:44:21 -0500 Subject: [PATCH 619/962] feat: Add support for addon `configuration_values` (#2345) Resolves undefined --- README.md | 4 +- examples/complete/README.md | 4 +- examples/complete/versions.tf | 2 +- examples/eks_managed_node_group/README.md | 4 +- examples/eks_managed_node_group/main.tf | 7 + examples/eks_managed_node_group/versions.tf | 2 +- examples/fargate_profile/README.md | 11 +- examples/fargate_profile/main.tf | 142 +------------------ examples/fargate_profile/versions.tf | 2 +- examples/karpenter/README.md | 12 +- examples/karpenter/main.tf | 133 ++--------------- examples/karpenter/versions.tf | 2 +- examples/outposts/README.md | 4 +- examples/outposts/versions.tf | 2 +- examples/self_managed_node_group/README.md | 4 +- examples/self_managed_node_group/versions.tf | 2 +- main.tf | 1 + modules/eks-managed-node-group/README.md | 4 +- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 +- modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 4 +- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 +- modules/self-managed-node-group/versions.tf | 2 +- versions.tf | 2 +- 26 files changed, 57 insertions(+), 307 deletions(-) diff --git a/README.md b/README.md index a0d1fce5c2..234a4b2aff 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | | [tls](#requirement\_tls) | >= 3.0 | @@ -231,7 +231,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | +| [aws](#provider\_aws) | >= 4.47 | | [kubernetes](#provider\_kubernetes) | >= 2.10 | | [tls](#provider\_tls) | >= 3.0 | diff --git a/examples/complete/README.md b/examples/complete/README.md index f6bac2e304..39b0eb3272 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -34,14 +34,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | +| [aws](#provider\_aws) | >= 4.47 | ## Modules diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index f2f8625d4b..aeb892f359 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index ecb37f380c..45fa191c3e 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -58,14 +58,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | +| [aws](#provider\_aws) | >= 4.47 | ## Modules diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 378496c7f2..4cac1e8eda 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -64,6 +64,13 @@ module "eks" { vpc-cni = { most_recent = true service_account_role_arn = module.vpc_cni_irsa.iam_role_arn + configuration_values = jsonencode({ + env = { + # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html + ENABLE_PREFIX_DELEGATION = true + WARM_PREFIX_TARGET = 1 + } + }) } } diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index f2f8625d4b..aeb892f359 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 71a9ee7ad0..1b48556ce1 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -20,7 +20,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | | [helm](#requirement\_helm) | >= 2.7 | | [null](#requirement\_null) | >= 3.0 | @@ -28,9 +28,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | -| [helm](#provider\_helm) | >= 2.7 | -| [null](#provider\_null) | >= 3.0 | +| [aws](#provider\_aws) | >= 4.47 | ## Modules @@ -44,12 +42,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| | [aws_iam_policy.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [helm_release.coredns](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [null_resource.modify_kube_dns](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.remove_default_coredns_deployment](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | -| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | ## Inputs diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index fa6c8bdb64..860ec55703 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -2,20 +2,6 @@ provider "aws" { region = local.region } -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } - } -} - data "aws_availability_zones" "available" {} locals { @@ -47,6 +33,11 @@ module "eks" { cluster_addons = { kube-proxy = {} vpc-cni = {} + coredns = { + configuration_values = jsonencode({ + computeType = "Fargate" + }) + } } vpc_id = module.vpc.vpc_id @@ -105,129 +96,6 @@ module "eks" { tags = local.tags } -################################################################################ -# Modify EKS CoreDNS Deployment -################################################################################ - -data "aws_eks_cluster_auth" "this" { - name = module.eks.cluster_name -} - -locals { - kubeconfig = yamlencode({ - apiVersion = "v1" - kind = "Config" - current-context = "terraform" - clusters = [{ - name = module.eks.cluster_name - cluster = { - certificate-authority-data = module.eks.cluster_certificate_authority_data - server = module.eks.cluster_endpoint - } - }] - contexts = [{ - name = "terraform" - context = { - cluster = module.eks.cluster_name - user = "terraform" - } - }] - users = [{ - name = "terraform" - user = { - token = data.aws_eks_cluster_auth.this.token - } - }] - }) -} - -# Separate resource so that this is only ever executed once -resource "null_resource" "remove_default_coredns_deployment" { - triggers = {} - - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - environment = { - KUBECONFIG = base64encode(local.kubeconfig) - } - - # We are removing the deployment provided by the EKS service and replacing it through the self-managed CoreDNS Helm addon - # However, we are maintaining the existing kube-dns service and annotating it for Helm to assume control - command = <<-EOT - kubectl --namespace kube-system delete deployment coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) - EOT - } -} - -resource "null_resource" "modify_kube_dns" { - triggers = {} - - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - environment = { - KUBECONFIG = base64encode(local.kubeconfig) - } - - # We are maintaining the existing kube-dns service and annotating it for Helm to assume control - command = <<-EOT - echo "Setting implicit dependency on ${module.eks.fargate_profiles["kube_system"].fargate_profile_pod_execution_role_arn}" - kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-name=coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) - kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-namespace=kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) - kubectl --namespace kube-system label --overwrite service kube-dns app.kubernetes.io/managed-by=Helm --kubeconfig <(echo $KUBECONFIG | base64 --decode) - EOT - } - - depends_on = [ - null_resource.remove_default_coredns_deployment - ] -} - -################################################################################ -# CoreDNS Helm Chart (self-managed) -################################################################################ - -data "aws_eks_addon_version" "this" { - for_each = toset(["coredns"]) - - addon_name = each.value - kubernetes_version = module.eks.cluster_version - most_recent = true -} - -resource "helm_release" "coredns" { - name = "coredns" - namespace = "kube-system" - create_namespace = false - description = "CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS Services" - chart = "coredns" - version = "1.19.4" - repository = "https://coredns.github.io/helm" - - # For EKS image repositories https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html - values = [ - <<-EOT - image: - repository: 602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/coredns - tag: ${data.aws_eks_addon_version.this["coredns"].version} - deployment: - name: coredns - annotations: - eks.amazonaws.com/compute-type: fargate - service: - name: kube-dns - annotations: - eks.amazonaws.com/compute-type: fargate - podAnnotations: - eks.amazonaws.com/compute-type: fargate - EOT - ] - - depends_on = [ - # Need to ensure the CoreDNS updates are performed before provisioning - null_resource.modify_kube_dns - ] -} - ################################################################################ # Supporting Resources ################################################################################ diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index 69ef526bd9..17db7d8988 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } helm = { source = "hashicorp/helm" diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index f5d4e2dcc3..3335cfc4b3 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -52,7 +52,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 1.14 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | @@ -62,11 +62,10 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | -| [aws.virginia](#provider\_aws.virginia) | >= 4.45 | +| [aws](#provider\_aws) | >= 4.47 | +| [aws.virginia](#provider\_aws.virginia) | >= 4.47 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 1.14 | -| [null](#provider\_null) | >= 3.0 | ## Modules @@ -80,17 +79,12 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| -| [helm_release.coredns](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.karpenter_node_template](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [null_resource.modify_kube_dns](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.remove_default_coredns_deployment](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | -| [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | -| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | ## Inputs diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index eeb44ac0fc..66674706fb 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -78,6 +78,16 @@ module "eks" { cluster_version = local.cluster_version cluster_endpoint_public_access = true + cluster_addons = { + kube-proxy = {} + vpc-cni = {} + coredns = { + configuration_values = jsonencode({ + computeType = "Fargate" + }) + } + } + vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets @@ -245,129 +255,6 @@ resource "kubectl_manifest" "karpenter_example_deployment" { ] } -################################################################################ -# Modify EKS CoreDNS Deployment -################################################################################ - -data "aws_eks_cluster_auth" "this" { - name = module.eks.cluster_name -} - -locals { - kubeconfig = yamlencode({ - apiVersion = "v1" - kind = "Config" - current-context = "terraform" - clusters = [{ - name = module.eks.cluster_name - cluster = { - certificate-authority-data = module.eks.cluster_certificate_authority_data - server = module.eks.cluster_endpoint - } - }] - contexts = [{ - name = "terraform" - context = { - cluster = module.eks.cluster_name - user = "terraform" - } - }] - users = [{ - name = "terraform" - user = { - token = data.aws_eks_cluster_auth.this.token - } - }] - }) -} - -# Separate resource so that this is only ever executed once -resource "null_resource" "remove_default_coredns_deployment" { - triggers = {} - - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - environment = { - KUBECONFIG = base64encode(local.kubeconfig) - } - - # We are removing the deployment provided by the EKS service and replacing it through the self-managed CoreDNS Helm addon - # However, we are maintaining the existing kube-dns service and annotating it for Helm to assume control - command = <<-EOT - kubectl --namespace kube-system delete deployment coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) - EOT - } -} - -resource "null_resource" "modify_kube_dns" { - triggers = {} - - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - environment = { - KUBECONFIG = base64encode(local.kubeconfig) - } - - # We are maintaining the existing kube-dns service and annotating it for Helm to assume control - command = <<-EOT - echo "Setting implicit dependency on ${module.eks.fargate_profiles["kube_system"].fargate_profile_pod_execution_role_arn}" - kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-name=coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) - kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-namespace=kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) - kubectl --namespace kube-system label --overwrite service kube-dns app.kubernetes.io/managed-by=Helm --kubeconfig <(echo $KUBECONFIG | base64 --decode) - EOT - } - - depends_on = [ - null_resource.remove_default_coredns_deployment - ] -} - -################################################################################ -# CoreDNS Helm Chart (self-managed) -################################################################################ - -data "aws_eks_addon_version" "this" { - for_each = toset(["coredns"]) - - addon_name = each.value - kubernetes_version = module.eks.cluster_version - most_recent = true -} - -resource "helm_release" "coredns" { - name = "coredns" - namespace = "kube-system" - create_namespace = false - description = "CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS Services" - chart = "coredns" - version = "1.19.4" - repository = "https://coredns.github.io/helm" - - # For EKS image repositories https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html - values = [ - <<-EOT - image: - repository: 602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/coredns - tag: ${data.aws_eks_addon_version.this["coredns"].version} - deployment: - name: coredns - annotations: - eks.amazonaws.com/compute-type: fargate - service: - name: kube-dns - annotations: - eks.amazonaws.com/compute-type: fargate - podAnnotations: - eks.amazonaws.com/compute-type: fargate - EOT - ] - - depends_on = [ - # Need to ensure the CoreDNS updates are performed before provisioning - null_resource.modify_kube_dns - ] -} - ################################################################################ # Supporting Resources ################################################################################ diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 7ae5250dce..cab7b21121 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/outposts/README.md b/examples/outposts/README.md index 20b60fd4e9..33684940ac 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -42,14 +42,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | +| [aws](#provider\_aws) | >= 4.47 | | [kubernetes](#provider\_kubernetes) | >= 2.10 | ## Modules diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf index f2f8625d4b..aeb892f359 100644 --- a/examples/outposts/versions.tf +++ b/examples/outposts/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 97deaa59ab..9d78d228b9 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -26,14 +26,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | +| [aws](#provider\_aws) | >= 4.47 | ## Modules diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index f2f8625d4b..aeb892f359 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/main.tf b/main.tf index 0001d55f6c..7566db516f 100644 --- a/main.tf +++ b/main.tf @@ -375,6 +375,7 @@ resource "aws_eks_addon" "this" { addon_name = try(each.value.name, each.key) addon_version = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) + configuration_values = try(each.value.configuration_values, null) preserve = try(each.value.preserve, null) resolve_conflicts = try(each.value.resolve_conflicts, "OVERWRITE") service_account_role_arn = try(each.value.service_account_role_arn, null) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index d4d4cc9b40..f05c31d5f6 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -65,13 +65,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | +| [aws](#provider\_aws) | >= 4.47 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 325eee94e1..55eff62b09 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 4ed9a6fd2d..cc0bab2a5b 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | +| [aws](#provider\_aws) | >= 4.47 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 325eee94e1..55eff62b09 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 71f4ff9455..eaf04f6554 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -100,13 +100,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | +| [aws](#provider\_aws) | >= 4.47 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 325eee94e1..55eff62b09 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 051aa4f4eb..f16a2514ea 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.45 | +| [aws](#requirement\_aws) | >= 4.47 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.45 | +| [aws](#provider\_aws) | >= 4.47 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 325eee94e1..55eff62b09 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } } } diff --git a/versions.tf b/versions.tf index fdc407c2ed..5da3d23ea7 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.45" + version = ">= 4.47" } tls = { source = "hashicorp/tls" From 678cf7fa812ed6cc5b68ef851de2c2e8ba576034 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 16 Dec 2022 19:44:59 +0000 Subject: [PATCH 620/962] chore(release): version 19.1.0 [skip ci] ## [19.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.4...v19.1.0) (2022-12-16) ### Features * Add support for addon `configuration_values` ([#2345](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2345)) ([3b62f6c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3b62f6c31604490fc19184e626e73873b296ecd1)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c3a58c3b7..9c798eb641 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.4...v19.1.0) (2022-12-16) + + +### Features + +* Add support for addon `configuration_values` ([#2345](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2345)) ([3b62f6c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3b62f6c31604490fc19184e626e73873b296ecd1)) + ### [19.0.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.3...v19.0.4) (2022-12-07) From 71b8ecaa87db89c454b2c9446ff3d7675e4dc5a7 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 17 Dec 2022 07:32:04 -0500 Subject: [PATCH 621/962] fix: Use IAM session context data source to resolve the identities role when using `assumed_role` (#2347) --- README.md | 1 + main.tf | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 234a4b2aff..423546a973 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_session_context.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_session_context) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [tls_certificate.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/data-sources/certificate) | data source | diff --git a/main.tf b/main.tf index 7566db516f..e94231e569 100644 --- a/main.tf +++ b/main.tf @@ -1,6 +1,14 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} +data "aws_iam_session_context" "current" { + # This data source provides information on the IAM source role of an STS assumed role + # For non-role ARNs, this data source simply passes the ARN through issuer ARN + # Ref https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2327#issuecomment-1355581682 + # Ref https://github.com/hashicorp/terraform-provider-aws/issues/28381 + arn = data.aws_caller_identity.current.arn +} + locals { create = var.create && var.putin_khuylo @@ -122,7 +130,7 @@ module "kms" { # Policy enable_default_policy = var.kms_key_enable_default_policy key_owners = var.kms_key_owners - key_administrators = coalescelist(var.kms_key_administrators, [data.aws_caller_identity.current.arn]) + key_administrators = coalescelist(var.kms_key_administrators, [data.aws_iam_session_context.current.issuer_arn]) key_users = concat([local.cluster_role], var.kms_key_users) key_service_users = var.kms_key_service_users source_policy_documents = var.kms_key_source_policy_documents From f0abb25ba265cc952fc2573884166c657385be85 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 17 Dec 2022 12:32:42 +0000 Subject: [PATCH 622/962] chore(release): version 19.1.1 [skip ci] ### [19.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.1.0...v19.1.1) (2022-12-17) ### Bug Fixes * Use IAM session context data source to resolve the identities role when using `assumed_role` ([#2347](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2347)) ([71b8eca](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/71b8ecaa87db89c454b2c9446ff3d7675e4dc5a7)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c798eb641..5441aeb115 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.1.0...v19.1.1) (2022-12-17) + + +### Bug Fixes + +* Use IAM session context data source to resolve the identities role when using `assumed_role` ([#2347](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2347)) ([71b8eca](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/71b8ecaa87db89c454b2c9446ff3d7675e4dc5a7)) + ## [19.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.0.4...v19.1.0) (2022-12-16) From ece82472a62d7674b10261701bc1ec697988638f Mon Sep 17 00:00:00 2001 From: Marvin Wichmann Date: Sat, 17 Dec 2022 21:06:12 +0100 Subject: [PATCH 623/962] chore: Fix typo for vpc-cni addon example (#2349) Fix typo for vpc-cni addon All environment variables have to be strings --- examples/eks_managed_node_group/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 4cac1e8eda..acdd39ac41 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -67,8 +67,8 @@ module "eks" { configuration_values = jsonencode({ env = { # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html - ENABLE_PREFIX_DELEGATION = true - WARM_PREFIX_TARGET = 1 + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" } }) } From 0751a0ca04d6303015e8a9c2f917956ea00d184b Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 18 Dec 2022 12:29:11 -0500 Subject: [PATCH 624/962] feat: Ensure all supported resources are tagged under `tag_specifications` on launch templates (#2352) Resolves undefined --- modules/eks-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index b844533abe..c59d553f19 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -270,7 +270,7 @@ resource "aws_launch_template" "this" { ram_disk_id = var.ram_disk_id dynamic "tag_specifications" { - for_each = toset(["instance", "volume", "network-interface"]) + for_each = toset(["instance", "volume", "elastic-gpu", "network-interface", "spot-instances-request"]) content { resource_type = tag_specifications.key diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 7632d24609..10752fb769 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -364,7 +364,7 @@ resource "aws_launch_template" "this" { ram_disk_id = var.ram_disk_id dynamic "tag_specifications" { - for_each = toset(["instance", "volume", "network-interface"]) + for_each = toset(["instance", "volume", "elastic-gpu", "network-interface", "spot-instances-request"]) content { resource_type = tag_specifications.key From 180434de6945ff9d5098f3d124e24971843253ae Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 18 Dec 2022 17:29:49 +0000 Subject: [PATCH 625/962] chore(release): version 19.2.0 [skip ci] ## [19.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.1.1...v19.2.0) (2022-12-18) ### Features * Ensure all supported resources are tagged under `tag_specifications` on launch templates ([#2352](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2352)) ([0751a0c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0751a0ca04d6303015e8a9c2f917956ea00d184b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5441aeb115..9b6e36fe5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.1.1...v19.2.0) (2022-12-18) + + +### Features + +* Ensure all supported resources are tagged under `tag_specifications` on launch templates ([#2352](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2352)) ([0751a0c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0751a0ca04d6303015e8a9c2f917956ea00d184b)) + ### [19.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.1.0...v19.1.1) (2022-12-17) From 5a270b7bf8de8c5846e91d72ffd9f594cbd8b921 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 18 Dec 2022 12:36:39 -0500 Subject: [PATCH 626/962] feat: Add additional port for `metrics-server` to recommended rules (#2353) --- node_groups.tf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/node_groups.tf b/node_groups.tf index 6561faec5a..fa7c1e2a62 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -115,6 +115,16 @@ locals { type = "ingress" self = true } + # metrics-server + ingress_cluster_8443_webhook = { + description = "Cluster API to node 4443/tcp webhook" + protocol = "tcp" + from_port = 4443 + to_port = 4443 + type = "ingress" + source_cluster_security_group = true + } + # Karpenter ingress_cluster_8443_webhook = { description = "Cluster API to node 8443/tcp webhook" protocol = "tcp" @@ -123,6 +133,7 @@ locals { type = "ingress" source_cluster_security_group = true } + # ALB controller, NGINX ingress_cluster_9443_webhook = { description = "Cluster API to node 9443/tcp webhook" protocol = "tcp" From 2b0a283a4efe92f39b16c54ac59aadb6c1c78540 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 18 Dec 2022 17:37:15 +0000 Subject: [PATCH 627/962] chore(release): version 19.3.0 [skip ci] ## [19.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.2.0...v19.3.0) (2022-12-18) ### Features * Add additional port for `metrics-server` to recommended rules ([#2353](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2353)) ([5a270b7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5a270b7bf8de8c5846e91d72ffd9f594cbd8b921)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b6e36fe5b..652386032d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.2.0...v19.3.0) (2022-12-18) + + +### Features + +* Add additional port for `metrics-server` to recommended rules ([#2353](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2353)) ([5a270b7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5a270b7bf8de8c5846e91d72ffd9f594cbd8b921)) + ## [19.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.1.1...v19.2.0) (2022-12-18) From 13a9542dadd29fa75fd76c2adcee9dd17dcffda4 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 18 Dec 2022 13:31:45 -0500 Subject: [PATCH 628/962] fix: Correct map name for security group rule 4443/tcp (#2354) --- node_groups.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_groups.tf b/node_groups.tf index fa7c1e2a62..65403300b3 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -116,7 +116,7 @@ locals { self = true } # metrics-server - ingress_cluster_8443_webhook = { + ingress_cluster_4443_webhook = { description = "Cluster API to node 4443/tcp webhook" protocol = "tcp" from_port = 4443 From 046c688be2c2440fafc8cef48db9f3bc8915058d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 18 Dec 2022 18:32:24 +0000 Subject: [PATCH 629/962] chore(release): version 19.3.1 [skip ci] ### [19.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.3.0...v19.3.1) (2022-12-18) ### Bug Fixes * Correct map name for security group rule 4443/tcp ([#2354](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2354)) ([13a9542](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/13a9542dadd29fa75fd76c2adcee9dd17dcffda4)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 652386032d..87372c145b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.3.0...v19.3.1) (2022-12-18) + + +### Bug Fixes + +* Correct map name for security group rule 4443/tcp ([#2354](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2354)) ([13a9542](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/13a9542dadd29fa75fd76c2adcee9dd17dcffda4)) + ## [19.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.2.0...v19.3.0) (2022-12-18) From 094ed1d5e461552a0a76bc019c36690fe0fc2dd5 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 19 Dec 2022 13:50:32 -0500 Subject: [PATCH 630/962] feat: Allow configuring which tags are passed on launch template tag specifications (#2360) Resolves undefined --- modules/eks-managed-node-group/README.md | 1 + modules/eks-managed-node-group/main.tf | 2 +- modules/eks-managed-node-group/variables.tf | 6 ++++++ modules/self-managed-node-group/README.md | 1 + modules/self-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/variables.tf | 6 ++++++ node_groups.tf | 2 ++ 7 files changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index f05c31d5f6..9a47d03b31 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -161,6 +161,7 @@ module "eks_managed_node_group" { | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | | [remote\_access](#input\_remote\_access) | Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false` | `any` | `{}` | no | | [subnet\_ids](#input\_subnet\_ids) | Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME` | `list(string)` | `null` | no | +| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface",
"spot-instances-request"
]
| no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [taints](#input\_taints) | The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group | `any` | `{}` | no | | [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the node group | `map(string)` | `{}` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index c59d553f19..dba2b58b5a 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -270,7 +270,7 @@ resource "aws_launch_template" "this" { ram_disk_id = var.ram_disk_id dynamic "tag_specifications" { - for_each = toset(["instance", "volume", "elastic-gpu", "network-interface", "spot-instances-request"]) + for_each = toset(var.tag_specifications) content { resource_type = tag_specifications.key diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index d2790e1acb..a2ef88f2c5 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -274,6 +274,12 @@ variable "launch_template_tags" { default = {} } +variable "tag_specifications" { + description = "The tags to apply to the resources during launch" + type = list(string) + default = ["instance", "volume", "network-interface", "spot-instances-request"] +} + ################################################################################ # EKS Managed Node Group ################################################################################ diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index f16a2514ea..89c86a507f 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -161,6 +161,7 @@ module "self_managed_node_group" { | [service\_linked\_role\_arn](#input\_service\_linked\_role\_arn) | The ARN of the service-linked role that the ASG will use to call other AWS services | `string` | `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones` | `list(string)` | `null` | no | | [suspended\_processes](#input\_suspended\_processes) | A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly | `list(string)` | `[]` | no | +| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface",
"spot-instances-request"
]
| no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [target\_group\_arns](#input\_target\_group\_arns) | A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing | `list(string)` | `[]` | no | | [termination\_policies](#input\_termination\_policies) | A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default` | `list(string)` | `[]` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 10752fb769..7ba6fe6e30 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -364,7 +364,7 @@ resource "aws_launch_template" "this" { ram_disk_id = var.ram_disk_id dynamic "tag_specifications" { - for_each = toset(["instance", "volume", "elastic-gpu", "network-interface", "spot-instances-request"]) + for_each = toset(var.tag_specifications) content { resource_type = tag_specifications.key diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index c2850b659e..b158d5dd2f 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -286,6 +286,12 @@ variable "launch_template_tags" { default = {} } +variable "tag_specifications" { + description = "The tags to apply to the resources during launch" + type = list(string) + default = ["instance", "volume", "network-interface", "spot-instances-request"] +} + ################################################################################ # Autoscaling group ################################################################################ diff --git a/node_groups.tf b/node_groups.tf index 65403300b3..fa79b2696a 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -297,6 +297,7 @@ module "eks_managed_node_group" { update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.eks_managed_node_group_defaults.update_launch_template_default_version, true) launch_template_description = try(each.value.launch_template_description, var.eks_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} EKS managed node group") launch_template_tags = try(each.value.launch_template_tags, var.eks_managed_node_group_defaults.launch_template_tags, {}) + tag_specifications = try(each.value.tag_specifications, var.eks_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface", "spot-instances-request"]) ebs_optimized = try(each.value.ebs_optimized, var.eks_managed_node_group_defaults.ebs_optimized, null) key_name = try(each.value.key_name, var.eks_managed_node_group_defaults.key_name, null) @@ -422,6 +423,7 @@ module "self_managed_node_group" { update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.self_managed_node_group_defaults.update_launch_template_default_version, true) launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") launch_template_tags = try(each.value.launch_template_tags, var.self_managed_node_group_defaults.launch_template_tags, {}) + tag_specifications = try(each.value.tag_specifications, var.eks_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface", "spot-instances-request"]) ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") From ee495077a4bd568e3900592e7cac7fe300b9d51d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 19 Dec 2022 18:51:23 +0000 Subject: [PATCH 631/962] chore(release): version 19.4.0 [skip ci] ## [19.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.3.1...v19.4.0) (2022-12-19) ### Features * Allow configuring which tags are passed on launch template tag specifications ([#2360](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2360)) ([094ed1d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/094ed1d5e461552a0a76bc019c36690fe0fc2dd5)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87372c145b..e7c9058a50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.3.1...v19.4.0) (2022-12-19) + + +### Features + +* Allow configuring which tags are passed on launch template tag specifications ([#2360](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2360)) ([094ed1d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/094ed1d5e461552a0a76bc019c36690fe0fc2dd5)) + ### [19.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.3.0...v19.3.1) (2022-12-18) From df7c57c199d9e9f54d9ed18fb7c1e3a47ad732ed Mon Sep 17 00:00:00 2001 From: Alex Swehla Date: Tue, 20 Dec 2022 09:36:27 -0500 Subject: [PATCH 632/962] fix: Correct `eks_managed_*` to `self_managed_*` for `tag_specification` argument (#2364) Fix incorrect variable name on self managed nodes Self managed nodes were pulling from the eks_managed_node_group_defaults and not self_managed_node_group_defaults for tag_specifications. --- node_groups.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_groups.tf b/node_groups.tf index fa79b2696a..3740c438d5 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -423,7 +423,7 @@ module "self_managed_node_group" { update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.self_managed_node_group_defaults.update_launch_template_default_version, true) launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") launch_template_tags = try(each.value.launch_template_tags, var.self_managed_node_group_defaults.launch_template_tags, {}) - tag_specifications = try(each.value.tag_specifications, var.eks_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface", "spot-instances-request"]) + tag_specifications = try(each.value.tag_specifications, var.self_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface", "spot-instances-request"]) ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") From 2a16c59a5b5b4d9d7bf452c532a24a596c4fd7a1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 20 Dec 2022 14:36:57 +0000 Subject: [PATCH 633/962] chore(release): version 19.4.1 [skip ci] ### [19.4.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.0...v19.4.1) (2022-12-20) ### Bug Fixes * Correct `eks_managed_*` to `self_managed_*` for `tag_specification` argument ([#2364](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2364)) ([df7c57c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/df7c57c199d9e9f54d9ed18fb7c1e3a47ad732ed)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7c9058a50..10c0cc61de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.4.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.0...v19.4.1) (2022-12-20) + + +### Bug Fixes + +* Correct `eks_managed_*` to `self_managed_*` for `tag_specification` argument ([#2364](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2364)) ([df7c57c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/df7c57c199d9e9f54d9ed18fb7c1e3a47ad732ed)) + ## [19.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.3.1...v19.4.0) (2022-12-19) From e391a99a7bd8209618fdb65cc09460673fbaf1bc Mon Sep 17 00:00:00 2001 From: Pavlos-Petros Tournaris Date: Tue, 20 Dec 2022 18:23:44 +0000 Subject: [PATCH 634/962] fix: Drop spot-instances-request from tag_specifications (#2363) * Drop spot-instances-request from tag_specifications * fix: Update documentation Co-authored-by: Bryant Biggs --- modules/eks-managed-node-group/README.md | 2 +- modules/eks-managed-node-group/variables.tf | 2 +- modules/self-managed-node-group/README.md | 2 +- modules/self-managed-node-group/variables.tf | 2 +- node_groups.tf | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 9a47d03b31..311b1b94ff 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -161,7 +161,7 @@ module "eks_managed_node_group" { | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | | [remote\_access](#input\_remote\_access) | Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false` | `any` | `{}` | no | | [subnet\_ids](#input\_subnet\_ids) | Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME` | `list(string)` | `null` | no | -| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface",
"spot-instances-request"
]
| no | +| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [taints](#input\_taints) | The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group | `any` | `{}` | no | | [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the node group | `map(string)` | `{}` | no | diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index a2ef88f2c5..36645e0d90 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -277,7 +277,7 @@ variable "launch_template_tags" { variable "tag_specifications" { description = "The tags to apply to the resources during launch" type = list(string) - default = ["instance", "volume", "network-interface", "spot-instances-request"] + default = ["instance", "volume", "network-interface"] } ################################################################################ diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 89c86a507f..8e04f81e3d 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -161,7 +161,7 @@ module "self_managed_node_group" { | [service\_linked\_role\_arn](#input\_service\_linked\_role\_arn) | The ARN of the service-linked role that the ASG will use to call other AWS services | `string` | `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones` | `list(string)` | `null` | no | | [suspended\_processes](#input\_suspended\_processes) | A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly | `list(string)` | `[]` | no | -| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface",
"spot-instances-request"
]
| no | +| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [target\_group\_arns](#input\_target\_group\_arns) | A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing | `list(string)` | `[]` | no | | [termination\_policies](#input\_termination\_policies) | A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default` | `list(string)` | `[]` | no | diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index b158d5dd2f..7e5d0cecb6 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -289,7 +289,7 @@ variable "launch_template_tags" { variable "tag_specifications" { description = "The tags to apply to the resources during launch" type = list(string) - default = ["instance", "volume", "network-interface", "spot-instances-request"] + default = ["instance", "volume", "network-interface"] } ################################################################################ diff --git a/node_groups.tf b/node_groups.tf index 3740c438d5..8d84d3339e 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -297,7 +297,7 @@ module "eks_managed_node_group" { update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.eks_managed_node_group_defaults.update_launch_template_default_version, true) launch_template_description = try(each.value.launch_template_description, var.eks_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} EKS managed node group") launch_template_tags = try(each.value.launch_template_tags, var.eks_managed_node_group_defaults.launch_template_tags, {}) - tag_specifications = try(each.value.tag_specifications, var.eks_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface", "spot-instances-request"]) + tag_specifications = try(each.value.tag_specifications, var.eks_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface"]) ebs_optimized = try(each.value.ebs_optimized, var.eks_managed_node_group_defaults.ebs_optimized, null) key_name = try(each.value.key_name, var.eks_managed_node_group_defaults.key_name, null) @@ -423,7 +423,7 @@ module "self_managed_node_group" { update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.self_managed_node_group_defaults.update_launch_template_default_version, true) launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") launch_template_tags = try(each.value.launch_template_tags, var.self_managed_node_group_defaults.launch_template_tags, {}) - tag_specifications = try(each.value.tag_specifications, var.self_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface", "spot-instances-request"]) + tag_specifications = try(each.value.tag_specifications, var.self_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface"]) ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") From 1f1cb3a5d7ba5e9b771fb70ed5ef251a9d12aa70 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 20 Dec 2022 18:24:17 +0000 Subject: [PATCH 635/962] chore(release): version 19.4.2 [skip ci] ### [19.4.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.1...v19.4.2) (2022-12-20) ### Bug Fixes * Drop spot-instances-request from tag_specifications ([#2363](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2363)) ([e391a99](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e391a99a7bd8209618fdb65cc09460673fbaf1bc)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10c0cc61de..b6e67565bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.4.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.1...v19.4.2) (2022-12-20) + + +### Bug Fixes + +* Drop spot-instances-request from tag_specifications ([#2363](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2363)) ([e391a99](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e391a99a7bd8209618fdb65cc09460673fbaf1bc)) + ### [19.4.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.0...v19.4.1) (2022-12-20) From 5ae6abcaf49c67d1f71bfd9840a70300655cd25d Mon Sep 17 00:00:00 2001 From: Attilio D'Onofrio <79836589+don-attilio@users.noreply.github.com> Date: Tue, 27 Dec 2022 20:24:57 +0100 Subject: [PATCH 636/962] docs: Update eks managed node group usage to v19 (#2366) --- docs/compute_resources.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 05d5ea1dd8..a7138e8c44 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -13,7 +13,7 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) documentation for service related details. -1. The module creates a custom launch template by default to ensure settings such as tags are propagated to instances. To use the default template provided by the AWS EKS managed node group service, disable the launch template creation and set the `launch_template_name` to an empty string: +1. The module creates a custom launch template by default to ensure settings such as tags are propagated to instances. Please note that many of the customization options listed [here](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/eks-managed-node-group#Inputs) are only available when a custom launch template is created. To use the default template provided by the AWS EKS managed node group service, disable the launch template creation by setting `use_custom_launch_template` to `false`: ```hcl eks_managed_node_groups = { From 460e43db77244ad3ca2e62514de712fb0cc2cd7a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 3 Jan 2023 09:20:45 -0500 Subject: [PATCH 637/962] fix: Use a version for to avoid GitHub API rate limiting on CI workflows (#2376) --- .github/workflows/pre-commit.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 06270c8a90..cb82671340 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -8,6 +8,7 @@ on: env: TERRAFORM_DOCS_VERSION: v0.16.0 + TFLINT_VERSION: v0.44.1 jobs: collectInputs: @@ -21,7 +22,7 @@ jobs: - name: Get root directories id: dirs - uses: clowdhaus/terraform-composite-actions/directories@v1.8.0 + uses: clowdhaus/terraform-composite-actions/directories@v1.8.3 preCommitMinVersions: name: Min TF pre-commit @@ -36,24 +37,26 @@ jobs: - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.2.0 + uses: clowdhaus/terraform-min-max@v1.2.4 with: directory: ${{ matrix.directory }} - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory != '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} + tflint-version: ${{ env.TFLINT_VERSION }} args: 'terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/*' - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory == '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} + tflint-version: ${{ env.TFLINT_VERSION }} args: 'terraform_validate --color=always --show-diff-on-failure --files $(ls *.tf)' preCommitMaxVersion: @@ -69,10 +72,12 @@ jobs: - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.2.0 + uses: clowdhaus/terraform-min-max@v1.2.4 - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 with: terraform-version: ${{ steps.minMax.outputs.maxVersion }} + tflint-version: ${{ env.TFLINT_VERSION }} terraform-docs-version: ${{ env.TERRAFORM_DOCS_VERSION }} + install-hcledit: true From 21e26e9045780ccf83cdbb0e1b6a2523602c6095 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 5 Jan 2023 07:53:37 -0500 Subject: [PATCH 638/962] chore: Update examples for Fargate high availability (#2378) --- examples/fargate_profile/main.tf | 68 +++++++++++++++++--------------- examples/karpenter/main.tf | 41 +++++++++++-------- examples/outposts/main.tf | 8 ---- 3 files changed, 61 insertions(+), 56 deletions(-) diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index 860ec55703..ac24389bc4 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -54,44 +54,48 @@ module "eks" { } } - fargate_profiles = { - example = { - name = "example" - selectors = [ - { - namespace = "backend" - labels = { - Application = "backend" + fargate_profiles = merge( + { + example = { + name = "example" + selectors = [ + { + namespace = "backend" + labels = { + Application = "backend" + } + }, + { + namespace = "app-*" + labels = { + Application = "app-wildcard" + } } - }, - { - namespace = "app-*" - labels = { - Application = "app-wildcard" - } - } - ] + ] - # Using specific subnets instead of the subnets supplied for the cluster itself - subnet_ids = [module.vpc.private_subnets[1]] + # Using specific subnets instead of the subnets supplied for the cluster itself + subnet_ids = [module.vpc.private_subnets[1]] - tags = { - Owner = "secondary" - } + tags = { + Owner = "secondary" + } - timeouts = { - create = "20m" - delete = "20m" + timeouts = { + create = "20m" + delete = "20m" + } + } + }, + { for i in range(3) : + "kube-system-${element(split("-", local.azs[i]), 2)}" => { + selectors = [ + { namespace = "kube-system" } + ] + # We want to create a profile per AZ for high availability + subnet_ids = element(module.vpc.private_subnets, i) } } - - kube_system = { - name = "kube-system" - selectors = [ - { namespace = "kube-system" } - ] - } - } + ) tags = local.tags } diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 66674706fb..497fab71ac 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -92,6 +92,10 @@ module "eks" { subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets + # Fargate profiles use the cluster primary security group so these are not utilized + create_cluster_security_group = false + create_node_security_group = false + manage_aws_auth_configmap = true aws_auth_roles = [ # We need to add in the Karpenter node IAM role for nodes launched by Karpenter @@ -105,21 +109,26 @@ module "eks" { }, ] - fargate_profiles = { - kube_system = { - name = "kube-system" - selectors = [ - { namespace = "kube-system" } - ] - } - - karpenter = { - name = "karpenter" - selectors = [ - { namespace = "karpenter" } - ] - } - } + fargate_profiles = merge( + { for i in range(3) : + "kube-system-${element(split("-", local.azs[i]), 2)}" => { + selectors = [ + { namespace = "kube-system" } + ] + # We want to create a profile per AZ for high availability + subnet_ids = [element(module.vpc.private_subnets, i)] + } + }, + { for i in range(3) : + "karpenter-${element(split("-", local.azs[i]), 2)}" => { + selectors = [ + { namespace = "karpenter" } + ] + # We want to create a profile per AZ for high availability + subnet_ids = [element(module.vpc.private_subnets, i)] + } + }, + ) tags = merge(local.tags, { # NOTE - if creating multiple security groups with this module, only tag the @@ -151,7 +160,7 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "v0.19.3" + version = "v0.21.1" set { name = "settings.aws.clusterName" diff --git a/examples/outposts/main.tf b/examples/outposts/main.tf index 50cbc9586d..5dc4797e0f 100644 --- a/examples/outposts/main.tf +++ b/examples/outposts/main.tf @@ -64,14 +64,6 @@ module "eks" { } } - self_managed_node_group_defaults = { - attach_cluster_primary_security_group = true - - iam_role_additional_policies = { - AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" - } - } - self_managed_node_groups = { outpost = { name = local.name From 0868d0583fbce02cd81b97e633137c1c2a666cf4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 5 Jan 2023 12:54:18 +0000 Subject: [PATCH 639/962] chore(release): version 19.4.3 [skip ci] ### [19.4.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.2...v19.4.3) (2023-01-05) ### Bug Fixes * Use a version for to avoid GitHub API rate limiting on CI workflows ([#2376](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2376)) ([460e43d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/460e43db77244ad3ca2e62514de712fb0cc2cd7a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6e67565bf..5e5be36e4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.4.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.2...v19.4.3) (2023-01-05) + + +### Bug Fixes + +* Use a version for to avoid GitHub API rate limiting on CI workflows ([#2376](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2376)) ([460e43d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/460e43db77244ad3ca2e62514de712fb0cc2cd7a)) + ### [19.4.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.1...v19.4.2) (2022-12-20) From 5015b429e656d927fb66f214c998713c6fc84755 Mon Sep 17 00:00:00 2001 From: Matt Parkes Date: Fri, 6 Jan 2023 03:32:23 +1000 Subject: [PATCH 640/962] feat: Ignore changes to labels and annotations on on `aws-auth` ConfigMap (#2380) Co-authored-by: Bryant Biggs Fixes https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2379 --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index e94231e569..4f0ffb6ab3 100644 --- a/main.tf +++ b/main.tf @@ -520,7 +520,7 @@ resource "kubernetes_config_map" "aws_auth" { lifecycle { # We are ignoring the data here since we will manage it with the resource below # This is only intended to be used in scenarios where the configmap does not exist - ignore_changes = [data] + ignore_changes = [data, metadata[0].labels, metadata[0].annotations] } } From 90aba626e1a600f1f5714b1cc8f8c649d26882c3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 5 Jan 2023 17:33:09 +0000 Subject: [PATCH 641/962] chore(release): version 19.5.0 [skip ci] ## [19.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.3...v19.5.0) (2023-01-05) ### Features * Ignore changes to labels and annotations on on `aws-auth` ConfigMap ([#2380](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2380)) ([5015b42](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5015b429e656d927fb66f214c998713c6fc84755)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e5be36e4a..c60f94c64b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.3...v19.5.0) (2023-01-05) + + +### Features + +* Ignore changes to labels and annotations on on `aws-auth` ConfigMap ([#2380](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2380)) ([5015b42](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5015b429e656d927fb66f214c998713c6fc84755)) + ### [19.4.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.2...v19.4.3) (2023-01-05) From 383493538748f1df844d40068cdde62579b79476 Mon Sep 17 00:00:00 2001 From: Dan Marshall Date: Thu, 5 Jan 2023 21:04:23 +0000 Subject: [PATCH 642/962] fix: AMI lookup should only happen when launch template is created (#2386) Fixes https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2385 --- modules/self-managed-node-group/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 7ba6fe6e30..6ec1534bd1 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -2,7 +2,7 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} data "aws_ami" "eks_default" { - count = var.create ? 1 : 0 + count = var.create && var.create_launch_template ? 1 : 0 filter { name = "name" From 1a33f9e78106f3bed37e0fa86677386880e389f0 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 5 Jan 2023 21:05:08 +0000 Subject: [PATCH 643/962] chore(release): version 19.5.1 [skip ci] ### [19.5.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.5.0...v19.5.1) (2023-01-05) ### Bug Fixes * AMI lookup should only happen when launch template is created ([#2386](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2386)) ([3834935](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/383493538748f1df844d40068cdde62579b79476)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c60f94c64b..5d3da078fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.5.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.5.0...v19.5.1) (2023-01-05) + + +### Bug Fixes + +* AMI lookup should only happen when launch template is created ([#2386](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2386)) ([3834935](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/383493538748f1df844d40068cdde62579b79476)) + ## [19.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.4.3...v19.5.0) (2023-01-05) From 674ab7b7f2aa5163b3d9dbdafb7cb1ec910cd9de Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 6 Jan 2023 11:03:13 -0500 Subject: [PATCH 644/962] chore: Correct example where a list data type is required (#2387) --- examples/fargate_profile/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index ac24389bc4..1e928558c9 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -92,7 +92,7 @@ module "eks" { { namespace = "kube-system" } ] # We want to create a profile per AZ for high availability - subnet_ids = element(module.vpc.private_subnets, i) + subnet_ids = [element(module.vpc.private_subnets, i)] } } ) From 0e3cb9a0afcea87e1da0c87ecc969ffb963619ea Mon Sep 17 00:00:00 2001 From: Fernando Miguel <168366+FernandoMiguel@users.noreply.github.com> Date: Fri, 13 Jan 2023 17:45:16 +0000 Subject: [PATCH 645/962] docs: Typos in MD (#2390) fix typos Signed-off-by: Fernando Miguel Signed-off-by: Fernando Miguel --- CHANGELOG.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d3da078fa..3b81849368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -249,7 +249,7 @@ All notable changes to this project will be documented in this file. ### Features -* Add support for specifiying NTP address to use private Amazon Time Sync Service ([#2125](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2125)) ([4543ab4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/4543ab454bea80b64381b88a631d955a7cfae247)) +* Add support for specifying NTP address to use private Amazon Time Sync Service ([#2125](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2125)) ([4543ab4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/4543ab454bea80b64381b88a631d955a7cfae247)) ## [18.25.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.24.1...v18.25.0) (2022-06-28) @@ -270,7 +270,7 @@ All notable changes to this project will be documented in this file. ### Features -* Add support for specifying conrol plane subnets separate from those used by node groups (data plane) ([#2113](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2113)) ([ebc91bc](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ebc91bcd37a919a350d872a5b235ccc2a79955a6)) +* Add support for specifying control plane subnets separate from those used by node groups (data plane) ([#2113](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2113)) ([ebc91bc](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ebc91bcd37a919a350d872a5b235ccc2a79955a6)) ## [18.23.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.22.0...v18.23.0) (2022-06-02) @@ -613,7 +613,7 @@ All notable changes to this project will be documented in this file. ### Bug Fixes -* Use the prefix_seperator var for node sg prefix ([#1751](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1751)) ([62879dd](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/62879dd81a69ba010f19ba9ece8392e1730b53e0)) +* Use the prefix_separator var for node sg prefix ([#1751](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1751)) ([62879dd](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/62879dd81a69ba010f19ba9ece8392e1730b53e0)) ### [18.0.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v18.0.3...v18.0.4) (2022-01-07) @@ -890,7 +890,7 @@ BREAKING CHANGES: - The variable `config_output_path` is renamed into `kubeconfig_output_path` for naming consistency. Please upgrade your configuration accordingly. NOTES: -- Since we now search only for Linux or Windows AMI if there is a worker groups for the corresponding plateform, we can now define different default root block device name for each plateform. Use locals `root_block_device_name` and `root_block_device_name_windows` to define your owns. +- Since we now search only for Linux or Windows AMI if there is a worker groups for the corresponding platform, we can now define different default root block device name for each platform. Use locals `root_block_device_name` and `root_block_device_name_windows` to define your owns. - The kubeconfig file permission is not world and group readable anymore. The default permission is now `600`. This value can be changed with the variable `var.kubeconfig_file_permission`. @@ -929,7 +929,7 @@ BUG FIXES: - Bump `terraform-aws-modules/http` provider version to support darwin arm64 release ([#1369](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1369)) DOCS: -- Use IRSA for Node Termination Handler IAM policy attachement in Instance Refresh example ([#1373](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1373)) +- Use IRSA for Node Termination Handler IAM policy attachment in Instance Refresh example ([#1373](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1373)) @@ -1002,7 +1002,7 @@ FEATURES: - Add permissions boundary to fargate execution IAM role ([#1108](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1108)) ENHANCEMENTS: -- Dont set -x in userdata to avoid printing sensitive informations in logs ([#1187](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1187)) +- Don't set -x in userdata to avoid printing sensitive information in logs ([#1187](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1187)) BUG FIXES: - Merge tags from Fargate profiles with common tags from cluster ([#1159](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1159)) @@ -1114,7 +1114,7 @@ BREAKING CHANGES: NOTES: - `credit_specification` for worker groups launch template can now be set to `null` so that we can use non burstable EC2 families - Starting in v12.1.0 the `cluster_id` output depends on the -`wait_for_cluster` null resource. This means that initialisation of the +`wait_for_cluster` null resource. This means that initialization of the kubernetes provider will be blocked until the cluster is really ready, if the module is set to manage the aws_auth ConfigMap and user followed the typical Usage Example. kubernetes resources in the same plan do not @@ -1220,7 +1220,7 @@ TESTS: - Remove unused kitchen test related stuff ([#787](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/787)) CI: -- Restrict sementic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) +- Restrict semantic PR to validate PR title only ([#804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/804)) [Unreleased]: https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v17.23.0...HEAD From 3e8e6053d76c8f158f83fcecf680665d604474ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l-Magnus=20Sl=C3=A5tto?= Date: Sat, 28 Jan 2023 13:45:05 +0100 Subject: [PATCH 646/962] chore: Upgrade CI workflows to use non-deprecated runtimes (#2420) --- .github/workflows/lock.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index d887a660fe..6b6c9cec02 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -8,7 +8,7 @@ jobs: lock: runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v3 + - uses: dessant/lock-threads@v4 with: github-token: ${{ secrets.GITHUB_TOKEN }} issue-comment: > diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 98c8b25851..81f674740b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,7 @@ jobs: fetch-depth: 0 - name: Release - uses: cycjimmy/semantic-release-action@v2 + uses: cycjimmy/semantic-release-action@v3 with: semantic_version: 18.0.0 extra_plugins: | From 26c2dffa91f28a891feaa90d417ff428f5342e4e Mon Sep 17 00:00:00 2001 From: Kentaro Sumi <48937821+Ken-Sumi1019@users.noreply.github.com> Date: Sat, 28 Jan 2023 21:51:59 +0900 Subject: [PATCH 647/962] docs: Align level of headlines (#2421) Co-authored-by: Bryant Biggs --- docs/UPGRADE-19.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/UPGRADE-19.0.md b/docs/UPGRADE-19.0.md index 4bef119746..a353ed529d 100644 --- a/docs/UPGRADE-19.0.md +++ b/docs/UPGRADE-19.0.md @@ -150,7 +150,7 @@ Self managed node groups on `v18.x` by default create a security group that does 3. New instances will launch without the self-managed node group security group, and prior instances will be terminated 4. Once the self-managed node group has cycled, the security group will be deleted -#### EKS Managed Node Groups +### EKS Managed Node Groups EKS managed node groups on `v18.x` by default create a security group that does not specify any rules. In `v19.x`, this security group has been removed due to the predominant lack of usage (most users rely on the the shared node security group). While still using version `v18.x` of your module definition, remove this security group from your node groups by setting `create_security_group = false`. From 8f0d94bc42a8427a062ec058f93a67b1c11d371d Mon Sep 17 00:00:00 2001 From: benjaminstokes Date: Sat, 28 Jan 2023 07:57:41 -0500 Subject: [PATCH 648/962] docs: Removed `vpc_id` from nodegroup sub-module docs; removed in v19.x (#2398) Co-authored-by: Bryant Biggs --- modules/eks-managed-node-group/README.md | 1 - modules/self-managed-node-group/README.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 311b1b94ff..476231ded8 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -12,7 +12,6 @@ module "eks_managed_node_group" { cluster_name = "my-cluster" cluster_version = "1.24" - vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] // The following variables are necessary if you decide to use the module outside of the parent EKS module context. diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 8e04f81e3d..f505e09fd3 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -14,8 +14,8 @@ module "self_managed_node_group" { cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" - vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + // The following variables are necessary if you decide to use the module outside of the parent EKS module context. // Without it, the security groups of the nodes are empty and thus won't join the cluster. vpc_security_group_ids = [ From 059dc0c67c2aebbf2c9a2f0a05856a823dd1b5a0 Mon Sep 17 00:00:00 2001 From: Adin Hodovic Date: Sat, 28 Jan 2023 14:02:12 +0100 Subject: [PATCH 649/962] feat: Add prometheus-adapter port 6443 to recommended sec groups (#2399) Co-authored-by: Bryant Biggs --- node_groups.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/node_groups.tf b/node_groups.tf index 8d84d3339e..85c61387c4 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -124,6 +124,15 @@ locals { type = "ingress" source_cluster_security_group = true } + # prometheus-adapter + ingress_cluster_6443_webhook = { + description = "Cluster API to node 6443/tcp webhook" + protocol = "tcp" + from_port = 6443 + to_port = 6443 + type = "ingress" + source_cluster_security_group = true + } # Karpenter ingress_cluster_8443_webhook = { description = "Cluster API to node 8443/tcp webhook" From 48d70801e99c8c36efe0e3bcc1ad463f34cc3ddc Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 28 Jan 2023 13:02:43 +0000 Subject: [PATCH 650/962] chore(release): version 19.6.0 [skip ci] ## [19.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.5.1...v19.6.0) (2023-01-28) ### Features * Add prometheus-adapter port 6443 to recommended sec groups ([#2399](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2399)) ([059dc0c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/059dc0c67c2aebbf2c9a2f0a05856a823dd1b5a0)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b81849368..8ab1f54022 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.5.1...v19.6.0) (2023-01-28) + + +### Features + +* Add prometheus-adapter port 6443 to recommended sec groups ([#2399](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2399)) ([059dc0c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/059dc0c67c2aebbf2c9a2f0a05856a823dd1b5a0)) + ### [19.5.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.5.0...v19.5.1) (2023-01-05) From a8e0de0b3f3a3e27232b35e9ed8814549273bced Mon Sep 17 00:00:00 2001 From: Simon Staszkiewicz Date: Sat, 28 Jan 2023 12:36:53 -0800 Subject: [PATCH 651/962] docs: Update README to include changes to variables (#2430) --- modules/eks-managed-node-group/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 476231ded8..d94f815075 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -17,7 +17,7 @@ module "eks_managed_node_group" { // The following variables are necessary if you decide to use the module outside of the parent EKS module context. // Without it, the security groups of the nodes are empty and thus won't join the cluster. cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id - cluster_security_group_id = module.eks.node_security_group_id + vpc_security_group_ids = [module.eks.node_security_group_id] // Note: `disk_size`, and `remote_access` can only be set when using the EKS managed node group default launch template // This module defaults to providing a custom launch template to allow for custom security groups, tag propagation, etc. From 68fe60f1c4e975d7f6f2c22ae891a32fd80a0156 Mon Sep 17 00:00:00 2001 From: Omer Levi Hevroni Date: Tue, 7 Feb 2023 14:04:45 +0200 Subject: [PATCH 652/962] feat: Allow to pass prefix for rule names (#2437) --- modules/karpenter/README.md | 1 + modules/karpenter/main.tf | 2 +- modules/karpenter/variables.tf | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index eaf04f6554..dfe0670844 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -172,6 +172,7 @@ No modules. | [queue\_kms\_master\_key\_id](#input\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no | | [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | | [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no | +| [rule\_name\_prefix](#input\_rule\_name\_prefix) | Prefix used for all event bridge rules | `string` | `"Karpenter"` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | ## Outputs diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 62640f0550..2133d49f9c 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -261,7 +261,7 @@ locals { resource "aws_cloudwatch_event_rule" "this" { for_each = { for k, v in local.events : k => v if local.enable_spot_termination } - name_prefix = "Karpenter${each.value.name}-" + name_prefix = "${var.rule_name_prefix}${each.value.name}-" description = each.value.description event_pattern = jsonencode(each.value.event_pattern) diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index a6327ba932..8fdefc2705 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -224,3 +224,13 @@ variable "create_instance_profile" { type = bool default = true } + +################################################################################ +# Event Bridge Rules +################################################################################ + +variable "rule_name_prefix" { + description = "Prefix used for all event bridge rules" + type = string + default = "Karpenter" +} From 4b7fe4637a048a52422ee04d6146136f48900695 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 7 Feb 2023 12:05:21 +0000 Subject: [PATCH 653/962] chore(release): version 19.7.0 [skip ci] ## [19.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.6.0...v19.7.0) (2023-02-07) ### Features * Allow to pass prefix for rule names ([#2437](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2437)) ([68fe60f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/68fe60f1c4e975d7f6f2c22ae891a32fd80a0156)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ab1f54022..173b79692f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.6.0...v19.7.0) (2023-02-07) + + +### Features + +* Allow to pass prefix for rule names ([#2437](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2437)) ([68fe60f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/68fe60f1c4e975d7f6f2c22ae891a32fd80a0156)) + ## [19.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.5.1...v19.6.0) (2023-01-28) From aaea28dfa054356e8ed1a655955ef78a4e1f134e Mon Sep 17 00:00:00 2001 From: felipewnp <34146218+felipewnp@users.noreply.github.com> Date: Thu, 9 Feb 2023 10:42:44 -0300 Subject: [PATCH 654/962] docs: Update UPGRADE-19.0.md (#2445) Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 2 +- docs/UPGRADE-19.0.md | 44 ++++++++++++++++++++--------------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d5886a6d32..071427d757 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.0 + rev: v1.77.1 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/docs/UPGRADE-19.0.md b/docs/UPGRADE-19.0.md index a353ed529d..dbdd9e6efb 100644 --- a/docs/UPGRADE-19.0.md +++ b/docs/UPGRADE-19.0.md @@ -5,13 +5,13 @@ Please consult the `examples` directory for reference example configurations. If ## List of backwards incompatible changes - The `cluster_id` output used to output the name of the cluster. This is due to the fact that the cluster name is a unique constraint and therefore its set as the unique identifier within Terraform's state map. However, starting with local EKS clusters created on Outposts, there is now an attribute returned from the `aws eks create-cluster` API named `id`. The `cluster_id` has been updated to return this value which means that for current, standard EKS clusters created in the AWS cloud, no value will be returned (at the time of this writing) for `cluster_id` and only local EKS clusters on Outposts will return a value that looks like a UUID/GUID. Users should switch all instances of `cluster_id` to use `cluster_name` before upgrading to v19. [Reference](https://github.com/hashicorp/terraform-provider-aws/issues/27560) -- Minimum supported version of Terraform AWS provider updated to v4.45 to support latest features provided via the resources utilized. +- Minimum supported version of Terraform AWS provider updated to v4.45 to support the latest features provided via the resources utilized. - Minimum supported version of Terraform updated to v1.0 -- Individual security group created per EKS managed node group or self managed node group has been removed. This configuration went mostly un-used and would often cause confusion ("Why is there an empty security group attached to my nodes?"). This functionality can easily be replicated by user's providing one or more externally created security groups to attach to nodes launched from the node group. -- Previously, `var.iam_role_additional_policies` (one for each of the following: cluster IAM role, EKS managed node group IAM role, self-managed node group IAM role, and Fargate Profile IAM role) accepted a list of strings. This worked well for policies that already existed but failed for policies being created at the same time as the cluster due to the well known issue of unknown values used in a `for_each` loop. To rectify this issue in `v19.x`, two changes were made: +- Individual security group created per EKS managed node group or self-managed node group has been removed. This configuration went mostly unused and would often cause confusion ("Why is there an empty security group attached to my nodes?"). This functionality can easily be replicated by user's providing one or more externally created security groups to attach to nodes launched from the node group. +- Previously, `var.iam_role_additional_policies` (one for each of the following: cluster IAM role, EKS managed node group IAM role, self-managed node group IAM role, and Fargate Profile IAM role) accepted a list of strings. This worked well for policies that already existed but failed for policies being created at the same time as the cluster due to the well-known issue of unknown values used in a `for_each` loop. To rectify this issue in `v19.x`, two changes were made: 1. `var.iam_role_additional_policies` was changed from type `list(string)` to type `map(string)` -> this is a breaking change. More information on managing this change can be found below, under `Terraform State Moves` 2. The logic used in the root module for this variable was changed to replace the use of `try()` with `lookup()`. More details on why can be found [here](https://github.com/clowdhaus/terraform-for-each-unknown) -- The cluster name has been removed from the Karpenter module event rule names. Due to the use of long cluster names appending to the provided naming scheme, the cluster name has moved to a `ClusterName` tag and the event rule name is now a prefix. This guarantees that users can have multiple instances of Karpenter withe their respective event rules/SQS queue without name collisions, while also still being able to identify which queues and event rules belong to which cluster. +- The cluster name has been removed from the Karpenter module event rule names. Due to the use of long cluster names appending to the provided naming scheme, the cluster name has moved to a `ClusterName` tag and the event rule name is now a prefix. This guarantees that users can have multiple instances of Karpenter with their respective event rules/SQS queue without name collisions, while also still being able to identify which queues and event rules belong to which cluster. ## Additional changes @@ -26,16 +26,16 @@ Please consult the `examples` directory for reference example configurations. If ### Modified -- `cluster_security_group_additional_rules` and `node_security_group_additional_rules` have been modified to use `lookup()` instead of `try()` to avoid the well known issue of [unknown values within a `for_each` loop](https://github.com/hashicorp/terraform/issues/4149) +- `cluster_security_group_additional_rules` and `node_security_group_additional_rules` have been modified to use `lookup()` instead of `try()` to avoid the well-known issue of [unknown values within a `for_each` loop](https://github.com/hashicorp/terraform/issues/4149) - Default cluster security group rules have removed egress rules for TCP/443 and TCP/10250 to node groups since the cluster primary security group includes a default rule for ALL to `0.0.0.0/0`/`::/0` - Default node security group rules have removed egress rules have been removed since the default security group settings have egress rule for ALL to `0.0.0.0/0`/`::/0` - `block_device_mappings` previously required a map of maps but has since changed to an array of maps. Users can remove the outer key for each block device mapping and replace the outermost map `{}` with an array `[]`. There are no state changes required for this change. -- `create_kms_key` previously defaulted to `false` and now defaults to `true`. Clusters created with this module now default to enabling secret encryption by default with a customer managed KMS key created by this module +- `create_kms_key` previously defaulted to `false` and now defaults to `true`. Clusters created with this module now default to enabling secret encryption by default with a customer-managed KMS key created by this module - `cluster_encryption_config` previously used a type of `list(any)` and now uses a type of `any` -> users can simply remove the outer `[`...`]` brackets on `v19.x` - `cluster_encryption_config` previously defaulted to `[]` and now defaults to `{resources = ["secrets"]}` to encrypt secrets by default -- `cluster_endpoint_public_access` previously defaulted to `true` and now defaults to `false`. Clusters created with this module now default to private only access to the cluster endpoint +- `cluster_endpoint_public_access` previously defaulted to `true` and now defaults to `false`. Clusters created with this module now default to private-only access to the cluster endpoint - `cluster_endpoint_private_access` previously defaulted to `false` and now defaults to `true` -- The addon configuration now sets `"OVERWRITE"` as the default value for `resolve_conflicts` to ease addon upgrade management. Users can opt out of this by instead setting `"NONE"` as the value for `resolve_conflicts` +- The addon configuration now sets `"OVERWRITE"` as the default value for `resolve_conflicts` to ease add-on upgrade management. Users can opt out of this by instead setting `"NONE"` as the value for `resolve_conflicts` - The `kms` module used has been updated from `v1.0.2` to `v1.1.0` - no material changes other than updated to latest - The default value for EKS managed node group `update_config` has been updated to the recommended `{ max_unavailable_percentage = 33 }` - The default value for the self-managed node group `instance_refresh` has been updated to the recommended: @@ -58,10 +58,10 @@ Please consult the `examples` directory for reference example configurations. If 1. Removed variables: - - `node_security_group_ntp_ipv4_cidr_block` - default security group settings have egress rule for ALL to `0.0.0.0/0`/`::/0` - - `node_security_group_ntp_ipv6_cidr_block` - default security group settings have egress rule for ALL to `0.0.0.0/0`/`::/0` + - `node_security_group_ntp_ipv4_cidr_block` - default security group settings have an egress rule for ALL to `0.0.0.0/0`/`::/0` + - `node_security_group_ntp_ipv6_cidr_block` - default security group settings have an egress rule for ALL to `0.0.0.0/0`/`::/0` - - Self managed node groups: + - Self-managed node groups: - `create_security_group` - `security_group_name` - `security_group_use_name_prefix` @@ -92,7 +92,7 @@ Please consult the `examples` directory for reference example configurations. If - `service_ipv6_cidr` for setting the IPv6 CIDR block for the Kubernetes service addresses - `node_security_group_enable_recommended_rules` for enabling recommended node security group rules for common access patterns - - Self managed node groups: + - Self-managed node groups: - `launch_template_id` for use when using an existing/externally created launch template (Ref: https://github.com/terraform-aws-modules/terraform-aws-autoscaling/pull/204) - `maintenance_options` - `private_dns_name_options` @@ -108,7 +108,7 @@ Please consult the `examples` directory for reference example configurations. If - 4. Removed outputs: - - Self managed node groups: + - Self-managed node groups: - `security_group_arn` - `security_group_id` - EKS managed node groups: @@ -121,18 +121,18 @@ Please consult the `examples` directory for reference example configurations. If 6. Added outputs: - - `cluster_name` - The `cluster_id` currently set by the AWS provider is actually the cluster name, but in the future this will change and there will be a distinction between the `cluster_name` and `cluster_id`. [Reference](https://github.com/hashicorp/terraform-provider-aws/issues/27560) + - `cluster_name` - The `cluster_id` currently set by the AWS provider is actually the cluster name, but in the future, this will change and there will be a distinction between the `cluster_name` and `cluster_id`. [Reference](https://github.com/hashicorp/terraform-provider-aws/issues/27560) ## Upgrade Migrations -1. Before upgrading your module definition to `v19.x`, please see below for both EKS managed node group(s) and self-managed node groups and removing the node group(s) security group prior to upgrading. +1. Before upgrading your module definition to `v19.x`, please see below for both EKS managed node group(s) and self-managed node groups and remove the node group(s) security group prior to upgrading. -### Self Managed Node Groups +### Self-Managed Node Groups -Self managed node groups on `v18.x` by default create a security group that does not specify any rules. In `v19.x`, this security group has been removed due to the predominant lack of usage (most users rely on the the shared node security group). While still using version `v18.x` of your module definition, remove this security group from your node groups by setting `create_security_group = false`. +Self-managed node groups on `v18.x` by default create a security group that does not specify any rules. In `v19.x`, this security group has been removed due to the predominant lack of usage (most users rely on the shared node security group). While still using version `v18.x` of your module definition, remove this security group from your node groups by setting `create_security_group = false`. - If you are currently utilizing this security group, it is recommended to create an additional security group that matches the rules/settings of the security group created by the node group, and specify that security group ID in `vpc_security_group_ids`. Once this is in place, you can proceed with the original security group removal. -- For most users, the security group is not used and can be safely removed. However, deployed instances will have the security group attached to nodes and require the security group to be disassociated before the security group can be deleted. Because instances are deployed via autoscaling groups, we cannot simply remove the security group from code and have those changes reflected on the instances. Instead, we have to update the code and then trigger the autoscaling groups to cycle the instances deployed so that new instances are provisioned without the security group attached. You can utilize the `instance_refresh` parameter of Autoscaling groups to force nodes to re-deploy when removing the security group since changes to launch templates automatically trigger an instance refresh. An example configuration is provided below. +- For most users, the security group is not used and can be safely removed. However, deployed instances will have the security group attached to nodes and require the security group to be disassociated before the security group can be deleted. Because instances are deployed via autoscaling groups, we cannot simply remove the security group from the code and have those changes reflected on the instances. Instead, we have to update the code and then trigger the autoscaling groups to cycle the instances deployed so that new instances are provisioned without the security group attached. You can utilize the `instance_refresh` parameter of Autoscaling groups to force nodes to re-deploy when removing the security group since changes to launch templates automatically trigger an instance refresh. An example configuration is provided below. - Add the following to either/or `self_managed_node_group_defaults` or the individual self-managed node group definitions: ```hcl create_security_group = false @@ -152,10 +152,10 @@ Self managed node groups on `v18.x` by default create a security group that does ### EKS Managed Node Groups -EKS managed node groups on `v18.x` by default create a security group that does not specify any rules. In `v19.x`, this security group has been removed due to the predominant lack of usage (most users rely on the the shared node security group). While still using version `v18.x` of your module definition, remove this security group from your node groups by setting `create_security_group = false`. +EKS managed node groups on `v18.x` by default create a security group that does not specify any rules. In `v19.x`, this security group has been removed due to the predominant lack of usage (most users rely on the shared node security group). While still using version `v18.x` of your module definition, remove this security group from your node groups by setting `create_security_group = false`. - If you are currently utilizing this security group, it is recommended to create an additional security group that matches the rules/settings of the security group created by the node group, and specify that security group ID in `vpc_security_group_ids`. Once this is in place, you can proceed with the original security group removal. -- EKS managed node groups rollout changes using a [rolling update strategy](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-update-behavior.html) that can be influenced through `update_config`. No additional changes are required for removing the the security group created by node groups (unlike self-managed node groups which should utilize the `instance_refresh` setting of Autoscaling groups). +- EKS managed node groups rollout changes using a [rolling update strategy](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-update-behavior.html) that can be influenced through `update_config`. No additional changes are required for removing the security group created by node groups (unlike self-managed node groups which should utilize the `instance_refresh` setting of Autoscaling groups). - Once `create_security_group = false` has been set, you can apply the changes which will: 1. Create a new launch template (version) without the EKS managed node group security group 2. Replace instances based on the `update_config` configuration settings @@ -236,7 +236,7 @@ EKS managed node groups on `v18.x` by default create a security group that does - } } - # Self Managed Node Group(s) + # Self-Managed Node Group(s) self_managed_node_group_defaults = { vpc_security_group_ids = [aws_security_group.additional.id] - iam_role_additional_policies = [aws_iam_policy.additional.arn] @@ -412,7 +412,7 @@ EKS managed node groups on `v18.x` by default create a security group that does ## Terraform State Moves -The following Terraform state move commands are optional but recommended if you are providing additional IAM policies that are to be attached to IAM roles created by this module (cluster IAM role, node group IAM role, Fargate profile IAM role). Because the resources affected are `aws_iam_role_policy_attachment`, in theory you could get away with simply applying the configuration and letting Terraform detach and re-attach the policies. However, during this brief period of update, you could experience permission failures as the policy is detached and re-attached and therefore the state move route is recommended. +The following Terraform state move commands are optional but recommended if you are providing additional IAM policies that are to be attached to IAM roles created by this module (cluster IAM role, node group IAM role, Fargate profile IAM role). Because the resources affected are `aws_iam_role_policy_attachment`, in theory, you could get away with simply applying the configuration and letting Terraform detach and re-attach the policies. However, during this brief period of update, you could experience permission failures as the policy is detached and re-attached, and therefore the state move route is recommended. Where `""` is specified, this should be replaced with the full ARN of the policy, and `""` should be replaced with the key used in the `iam_role_additional_policies` map for the associated policy. For example, if you have the following`v19.x` configuration: From 07653b62496962d62fad7eaa90b2cfaa2de9db09 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 9 Feb 2023 13:03:01 -0500 Subject: [PATCH 655/962] docs: Update documentation for 1.24 changes to containerd, ensure examples are functioning as intended (#2463) Resolves undefined --- docs/compute_resources.md | 32 +++------------------- docs/faq.md | 5 ++++ docs/user_data.md | 19 ------------- examples/eks_managed_node_group/README.md | 30 +------------------- examples/eks_managed_node_group/main.tf | 31 ++++----------------- examples/self_managed_node_group/README.md | 3 +- examples/self_managed_node_group/main.tf | 27 ++++-------------- examples/user_data/main.tf | 7 ++--- 8 files changed, 26 insertions(+), 128 deletions(-) diff --git a/docs/compute_resources.md b/docs/compute_resources.md index a7138e8c44..5e1d59f03c 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -36,28 +36,7 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ } ``` -3. Users have limited support to extend the user data that is pre-pended to the user data provided by the AWS EKS Managed Node Group service: - -```hcl - eks_managed_node_groups = { - prepend_userdata = { - # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 - pre_bootstrap_user_data = <<-EOT - #!/bin/bash - set -ex - cat <<-EOF > /etc/profile.d/bootstrap.sh - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - export KUBELET_EXTRA_ARGS="--max-pods=110" - EOF - # Source extra environment variables in bootstrap script - sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh - EOT - } - } -``` - -4. Bottlerocket OS is supported in a similar manner. However, note that the user data for Bottlerocket OS uses the TOML format: +3. Bottlerocket OS is supported in a similar manner. However, note that the user data for Bottlerocket OS uses the TOML format: ```hcl eks_managed_node_groups = { @@ -74,7 +53,7 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ } ``` -5. When using a custom AMI, the AWS EKS Managed Node Group service will NOT inject the necessary bootstrap script into the supplied user data. Users can elect to provide their own user data to bootstrap and connect or opt in to use the module provided user data: +4. When using a custom AMI, the AWS EKS Managed Node Group service will NOT inject the necessary bootstrap script into the supplied user data. Users can elect to provide their own user data to bootstrap and connect or opt in to use the module provided user data: ```hcl eks_managed_node_groups = { @@ -86,11 +65,8 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ # Note: this assumes the AMI provided is an EKS optimized AMI derivative enable_bootstrap_user_data = true - bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" - pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export FOO=bar EOT # Because we have full control over the user data supplied, we can also run additional @@ -102,7 +78,7 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ } ``` -6. There is similar support for Bottlerocket OS: +5. There is similar support for Bottlerocket OS: ```hcl eks_managed_node_groups = { diff --git a/docs/faq.md b/docs/faq.md index c53b027dca..215cdb2086 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,11 +1,16 @@ # Frequently Asked Questions +- [Setting `disk_size` or `remote_access` does not make any changes](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#Settings-disk_size-or-remote_access-does-not-make-any-changes) - [I received an error: `expect exactly one securityGroup tagged with kubernetes.io/cluster/ ...`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#i-received-an-error-expect-exactly-one-securitygroup-tagged-with-kubernetesioclustername-) - [Why are nodes not being registered?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-nodes-not-being-registered) - [Why are there no changes when a node group's `desired_size` is modified?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-there-no-changes-when-a-node-groups-desired_size-is-modified) - [How can I deploy Windows based nodes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-can-i-deploy-windows-based-nodes) - [How do I access compute resource attributes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-do-i-access-compute-resource-attributes) +### Setting `disk_size` or `remote_access` does not make any changes + +`disk_size`, and `remote_access` can only be set when using the EKS managed node group default launch template. This module defaults to providing a custom launch template to allow for custom security groups, tag propagation, etc. If you wish to forgo the custom launch template route, you can set `use_custom_launch_template = false` and then you can set `disk_size` and `remote_access`. + ### I received an error: `expect exactly one securityGroup tagged with kubernetes.io/cluster/ ...` By default, EKS creates a cluster primary security group that is created outside of the module and the EKS service adds the tag `{ "kubernetes.io/cluster/" = "owned" }`. This on its own does not cause any conflicts for addons such as the AWS Load Balancer Controller until users decide to attach both the cluster primary security group and the shared node security group created by the module (by setting `attach_cluster_primary_security_group = true`). The issue is not with having multiple security groups in your account with this tag key:value combination, but having multiple security groups with this tag key:value combination attached to nodes in the same cluster. There are a few ways to resolve this depending on your use case/intentions: diff --git a/docs/user_data.md b/docs/user_data.md index 4ce9a13ac4..7b236944ff 100644 --- a/docs/user_data.md +++ b/docs/user_data.md @@ -48,25 +48,6 @@ When using an EKS managed node group, users have 2 primary routes for interactin | ℹ️ When using bottlerocket as the desired platform, since the user data for bottlerocket is TOML, all configurations are merged in the one file supplied as user data. Therefore, `pre_bootstrap_user_data` and `post_bootstrap_user_data` are not valid since the bottlerocket OS handles when various settings are applied. If you wish to supply additional configuration settings when using bottlerocket, supply them via the `bootstrap_extra_args` variable. For the linux platform, `bootstrap_extra_args` are settings that will be supplied to the [AWS EKS Optimized AMI bootstrap script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh#L14) such as kubelet extra args, etc. See the [bottlerocket GitHub repository documentation](https://github.com/bottlerocket-os/bottlerocket#description-of-settings) for more details on what settings can be supplied via the `bootstrap_extra_args` variable. | | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -#### ⚠️ Caveat - -Since the EKS Managed Node Group service provides the necessary bootstrap user data to nodes (unless an `ami_id` is provided), users do not have direct access to settings/variables provided by the EKS optimized AMI [`bootstrap.sh` script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh). Currently, users must employ work-arounds to influence the `bootstrap.sh` script. For example, to enable `containerd` on EKS Managed Node Groups, users can supply the following user data. You can learn more about this issue [here](https://github.com/awslabs/amazon-eks-ami/issues/844): - -```hcl - # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 - pre_bootstrap_user_data = <<-EOT - #!/bin/bash - set -ex - cat <<-EOF > /etc/profile.d/bootstrap.sh - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - export KUBELET_EXTRA_ARGS="--max-pods=110" - EOF - # Source extra environment variables in bootstrap script - sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh - EOT -``` - ### Self Managed Node Group Self managed node groups require users to provide the necessary bootstrap user data. Users can elect to use the user data template provided by the module for their platform/OS or provide their own user data template for rendering by the module. diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 45fa191c3e..9328a05e5b 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -12,34 +12,6 @@ Configuration in this directory creates an AWS EKS cluster with various EKS Mana See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for further details. -## Container Runtime & User Data - -When using the default AMI provided by the EKS Managed Node Group service (i.e. - not specifying a value for `ami_id`), users should be aware of the limitations of configuring the node bootstrap process via user data. Due to not having direct access to the bootstrap.sh script invocation and therefore its configuration flags (this is provided by the EKS Managed Node Group service in the node user data), a workaround for ensuring the appropriate configuration settings is shown below. The following example shows how to inject configuration variables ahead of the merged user data provided by the EKS Managed Node Group service as well as how to enable the containerd runtime using this approach. More details can be found [here](https://github.com/awslabs/amazon-eks-ami/issues/844). - -```hcl - ... - # Demo of containerd usage when not specifying a custom AMI ID - # (merged into user data before EKS MNG provided user data) - containerd = { - name = "containerd" - - # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 - pre_bootstrap_user_data = <<-EOT - #!/bin/bash - set -ex - cat <<-EOF > /etc/profile.d/bootstrap.sh - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - export KUBELET_EXTRA_ARGS="--max-pods=110" - EOF - # Source extra environment variables in bootstrap script - sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh - sed -i 's/KUBELET_EXTRA_ARGS=$2/KUBELET_EXTRA_ARGS="$2 $KUBELET_EXTRA_ARGS"/' /etc/eks/bootstrap.sh - EOT - } - ... -``` - ## Usage To run this example you need to execute: @@ -71,7 +43,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| -| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 1.1 | +| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 1.5 | | [eks](#module\_eks) | ../.. | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index acdd39ac41..2373d88d6f 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -181,25 +181,6 @@ module "eks" { instance_types = ["t4g.medium"] } - # Demo of containerd usage when not specifying a custom AMI ID - # (merged into user data before EKS MNG provided user data) - containerd = { - name = "containerd" - - # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 - pre_bootstrap_user_data = <<-EOT - #!/bin/bash - set -ex - cat <<-EOF > /etc/profile.d/bootstrap.sh - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - export KUBELET_EXTRA_ARGS="--max-pods=110" - EOF - # Source extra environment variables in bootstrap script - sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh - EOT - } - # Complete complete = { name = "complete-eks-mng" @@ -213,11 +194,9 @@ module "eks" { ami_id = data.aws_ami.eks_default.image_id enable_bootstrap_user_data = true - bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export FOO=bar EOT post_bootstrap_user_data = <<-EOT @@ -259,7 +238,7 @@ module "eks" { iops = 3000 throughput = 150 encrypted = true - kms_key_id = module.ebs_kms_key.key_id + kms_key_id = module.ebs_kms_key.key_arn delete_on_termination = true } } @@ -315,6 +294,7 @@ module "vpc" { public_subnet_ipv6_prefixes = [0, 1, 2] private_subnet_ipv6_prefixes = [3, 4, 5] + intra_subnet_ipv6_prefixes = [6, 7, 8] enable_nat_gateway = true single_nat_gateway = true @@ -355,7 +335,7 @@ module "vpc_cni_irsa" { module "ebs_kms_key" { source = "terraform-aws-modules/kms/aws" - version = "~> 1.1" + version = "~> 1.5" description = "Customer managed key to encrypt EKS managed node group volumes" @@ -363,7 +343,8 @@ module "ebs_kms_key" { key_administrators = [ data.aws_caller_identity.current.arn ] - key_service_users = [ + + key_service_roles_for_autoscaling = [ # required for the ASG to manage encrypted volumes for nodes "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the cluster / persistentvolume-controller to create encrypted PVCs diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 9d78d228b9..d6944ac76d 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -39,7 +39,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| -| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 1.1 | +| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 1.5 | | [eks](#module\_eks) | ../.. | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | @@ -48,7 +48,6 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| -| [aws_ec2_capacity_reservation.targeted](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ec2_capacity_reservation) | resource | | [aws_iam_policy.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index fe7b5d85c8..7f1993d05a 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -182,12 +182,10 @@ module "eks" { max_size = 7 desired_size = 1 - ami_id = data.aws_ami.eks_default.id - bootstrap_extra_args = "--kubelet-extra-args '--max-pods=110'" + ami_id = data.aws_ami.eks_default.id pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export FOO=bar EOT post_bootstrap_user_data = <<-EOT @@ -212,7 +210,7 @@ module "eks" { iops = 3000 throughput = 150 encrypted = true - kms_key_id = module.ebs_kms_key.key_id + kms_key_id = module.ebs_kms_key.key_arn delete_on_termination = true } } @@ -225,12 +223,6 @@ module "eks" { instance_metadata_tags = "disabled" } - capacity_reservation_specification = { - capacity_reservation_target = { - capacity_reservation_id = aws_ec2_capacity_reservation.targeted.id - } - } - create_iam_role = true iam_role_name = "self-managed-node-group-complete-example" iam_role_use_name_prefix = false @@ -325,7 +317,7 @@ module "key_pair" { module "ebs_kms_key" { source = "terraform-aws-modules/kms/aws" - version = "~> 1.1" + version = "~> 1.5" description = "Customer managed key to encrypt EKS managed node group volumes" @@ -333,7 +325,8 @@ module "ebs_kms_key" { key_administrators = [ data.aws_caller_identity.current.arn ] - key_service_users = [ + + key_service_roles_for_autoscaling = [ # required for the ASG to manage encrypted volumes for nodes "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the cluster / persistentvolume-controller to create encrypted PVCs @@ -346,14 +339,6 @@ module "ebs_kms_key" { tags = local.tags } -resource "aws_ec2_capacity_reservation" "targeted" { - instance_type = "m6i.large" - instance_platform = "Linux/UNIX" - availability_zone = "${local.region}a" - instance_count = 1 - instance_match_criteria = "targeted" -} - resource "aws_iam_policy" "additional" { name = "${local.name}-additional" description = "Example usage of node additional policy" diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index cb565ba04c..d7d513190e 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -19,7 +19,7 @@ module "eks_mng_linux_additional" { source = "../../modules/_user_data" pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false EOT } @@ -34,11 +34,10 @@ module "eks_mng_linux_custom_ami" { enable_bootstrap_user_data = true pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export FOO=bar EOT - bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20 --instance-type t3a.large'" + bootstrap_extra_args = "--kubelet-extra-args '--instance-type t3a.large'" post_bootstrap_user_data = <<-EOT echo "All done" From c4a4b8afe3d1e89117573e9e04aea08871a069dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20W=C3=B6hrl?= Date: Wed, 15 Feb 2023 22:02:27 +0100 Subject: [PATCH 656/962] feat: Add auto discovery permission of cluster endpoint to Karpenter role (#2451) Co-authored-by: Bryant Biggs --- modules/karpenter/main.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 2133d49f9c..343e38b256 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -131,6 +131,11 @@ data "aws_iam_policy_document" "irsa" { resources = var.irsa_ssm_parameter_arns } + statement { + actions = ["eks:DescribeCluster"] + resources = ["arn:${local.partition}:eks:*:${local.account_id}:cluster/${var.cluster_name}"] + } + statement { actions = ["iam:PassRole"] resources = [var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn] From f7c25705de9d6fb8f08e12f8181caf3a81f1fb9d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 15 Feb 2023 21:03:01 +0000 Subject: [PATCH 657/962] chore(release): version 19.8.0 [skip ci] ## [19.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.7.0...v19.8.0) (2023-02-15) ### Features * Add auto discovery permission of cluster endpoint to Karpenter role ([#2451](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2451)) ([c4a4b8a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c4a4b8afe3d1e89117573e9e04aea08871a069dc)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 173b79692f..9b5b79a70b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.7.0...v19.8.0) (2023-02-15) + + +### Features + +* Add auto discovery permission of cluster endpoint to Karpenter role ([#2451](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2451)) ([c4a4b8a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c4a4b8afe3d1e89117573e9e04aea08871a069dc)) + ## [19.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.6.0...v19.7.0) (2023-02-07) From 78027f37e43c79748cd7528d3803122cb8072ed7 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 17 Feb 2023 07:28:03 -0500 Subject: [PATCH 658/962] feat: Add support for enabling addons before data plane compute is created (#2478) --- README.md | 5 ++++ examples/eks_managed_node_group/main.tf | 1 + main.tf | 27 ++++++++++++++++-- node_groups.tf | 37 +++++++++++++++++++------ outputs.tf | 4 +-- variables.tf | 6 ++++ versions.tf | 4 +++ 7 files changed, 71 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 423546a973..f265acf94a 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [terraform](#requirement\_terraform) | >= 1.0 | | [aws](#requirement\_aws) | >= 4.47 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | +| [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | ## Providers @@ -233,6 +234,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|---------| | [aws](#provider\_aws) | >= 4.47 | | [kubernetes](#provider\_kubernetes) | >= 2.10 | +| [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | ## Modules @@ -250,6 +252,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|------| | [aws_cloudwatch_log_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | | [aws_ec2_tag.cluster_primary_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ec2_tag) | resource | +| [aws_eks_addon.before_compute](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_addon.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | | [aws_eks_identity_provider_config.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_identity_provider_config) | resource | @@ -266,6 +269,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | | [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource | +| [time_sleep.this](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | @@ -326,6 +330,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [create\_kms\_key](#input\_create\_kms\_key) | Controls if a KMS key for cluster encryption should be created | `bool` | `true` | no | | [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | | [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | +| [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed nodegroup(s), self-managed nodegroup(s), Fargate profile(s)) | `string` | `"30s"` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 2373d88d6f..75210c00ec 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -63,6 +63,7 @@ module "eks" { } vpc-cni = { most_recent = true + before_compute = true service_account_role_arn = module.vpc_cni_irsa.iam_role_arn configuration_values = jsonencode({ env = { diff --git a/main.tf b/main.tf index 4f0ffb6ab3..604f90ce86 100644 --- a/main.tf +++ b/main.tf @@ -85,7 +85,8 @@ resource "aws_eks_cluster" "this" { aws_iam_role_policy_attachment.this, aws_security_group_rule.cluster, aws_security_group_rule.node, - aws_cloudwatch_log_group.this + aws_cloudwatch_log_group.this, + aws_iam_policy.cni_ipv6_policy, ] } @@ -377,7 +378,7 @@ resource "aws_iam_policy" "cluster_encryption" { resource "aws_eks_addon" "this" { # Not supported on outposts - for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } + for_each = { for k, v in var.cluster_addons : k => v if !try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } cluster_name = aws_eks_cluster.this[0].name addon_name = try(each.value.name, each.key) @@ -403,6 +404,28 @@ resource "aws_eks_addon" "this" { tags = var.tags } +resource "aws_eks_addon" "before_compute" { + # Not supported on outposts + for_each = { for k, v in var.cluster_addons : k => v if try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } + + cluster_name = aws_eks_cluster.this[0].name + addon_name = try(each.value.name, each.key) + + addon_version = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) + configuration_values = try(each.value.configuration_values, null) + preserve = try(each.value.preserve, null) + resolve_conflicts = try(each.value.resolve_conflicts, "OVERWRITE") + service_account_role_arn = try(each.value.service_account_role_arn, null) + + timeouts { + create = try(each.value.timeouts.create, var.cluster_addons_timeouts.create, null) + update = try(each.value.timeouts.update, var.cluster_addons_timeouts.update, null) + delete = try(each.value.timeouts.delete, var.cluster_addons_timeouts.delete, null) + } + + tags = var.tags +} + data "aws_eks_addon_version" "this" { for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } diff --git a/node_groups.tf b/node_groups.tf index 85c61387c4..0dacc1ed23 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -19,6 +19,25 @@ locals { } } +# This sleep resource is used to provide a timed gap between the cluster creation and the downstream dependencies +# that consume the outputs from here. Any of the values that are used as triggers can be used in dependencies +# to ensure that the downstream resources are created after both the cluster is ready and the sleep time has passed. +# This was primarily added to give addons that need to be configured BEFORE data plane compute resources +# enough time to create and configure themselves before the data plane compute resources are created. +resource "time_sleep" "this" { + count = var.create ? 1 : 0 + + create_duration = var.dataplane_wait_duration + + triggers = { + cluster_name = aws_eks_cluster.this[0].name + cluster_endpoint = aws_eks_cluster.this[0].endpoint + cluster_version = aws_eks_cluster.this[0].version + + cluster_certificate_authority_data = aws_eks_cluster.this[0].certificate_authority[0].data + } +} + ################################################################################ # EKS IPV6 CNI Policy # TODO - hopefully AWS releases a managed policy which can replace this @@ -220,7 +239,7 @@ module "fargate_profile" { create = try(each.value.create, true) # Fargate Profile - cluster_name = aws_eks_cluster.this[0].name + cluster_name = time_sleep.this[0].triggers["cluster_name"] cluster_ip_family = var.cluster_ip_family name = try(each.value.name, each.key) subnet_ids = try(each.value.subnet_ids, var.fargate_profile_defaults.subnet_ids, var.subnet_ids) @@ -255,8 +274,8 @@ module "eks_managed_node_group" { create = try(each.value.create, true) - cluster_name = aws_eks_cluster.this[0].name - cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, aws_eks_cluster.this[0].version) + cluster_name = time_sleep.this[0].triggers["cluster_name"] + cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"]) cluster_ip_family = var.cluster_ip_family # EKS Managed Node Group @@ -286,8 +305,8 @@ module "eks_managed_node_group" { # User data platform = try(each.value.platform, var.eks_managed_node_group_defaults.platform, "linux") - cluster_endpoint = try(aws_eks_cluster.this[0].endpoint, "") - cluster_auth_base64 = try(aws_eks_cluster.this[0].certificate_authority[0].data, "") + cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") + cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr enable_bootstrap_user_data = try(each.value.enable_bootstrap_user_data, var.eks_managed_node_group_defaults.enable_bootstrap_user_data, false) pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.eks_managed_node_group_defaults.pre_bootstrap_user_data, "") @@ -362,7 +381,7 @@ module "self_managed_node_group" { create = try(each.value.create, true) - cluster_name = aws_eks_cluster.this[0].name + cluster_name = time_sleep.this[0].triggers["cluster_name"] cluster_ip_family = var.cluster_ip_family # Autoscaling Group @@ -415,8 +434,8 @@ module "self_managed_node_group" { # User data platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, "linux") - cluster_endpoint = try(aws_eks_cluster.this[0].endpoint, "") - cluster_auth_base64 = try(aws_eks_cluster.this[0].certificate_authority[0].data, "") + cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") + cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.self_managed_node_group_defaults.pre_bootstrap_user_data, "") post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "") bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "") @@ -436,7 +455,7 @@ module "self_managed_node_group" { ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") - cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, aws_eks_cluster.this[0].version) + cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"]) instance_type = try(each.value.instance_type, var.self_managed_node_group_defaults.instance_type, "m6i.large") key_name = try(each.value.key_name, var.self_managed_node_group_defaults.key_name, null) diff --git a/outputs.tf b/outputs.tf index a0795ec736..f0cb797e14 100644 --- a/outputs.tf +++ b/outputs.tf @@ -19,12 +19,12 @@ output "cluster_endpoint" { output "cluster_id" { description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" - value = try(aws_eks_cluster.this[0].cluster_id, null) + value = try(aws_eks_cluster.this[0].cluster_id, "") } output "cluster_name" { description = "The name of the EKS cluster" - value = try(aws_eks_cluster.this[0].name, null) + value = try(aws_eks_cluster.this[0].name, "") } output "cluster_oidc_issuer_url" { diff --git a/variables.tf b/variables.tf index d0047ff1c0..dc96661810 100644 --- a/variables.tf +++ b/variables.tf @@ -460,6 +460,12 @@ variable "cluster_encryption_policy_tags" { default = {} } +variable "dataplane_wait_duration" { + description = "Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed nodegroup(s), self-managed nodegroup(s), Fargate profile(s))" + type = string + default = "30s" +} + ################################################################################ # EKS Addons ################################################################################ diff --git a/versions.tf b/versions.tf index 5da3d23ea7..dfe867e876 100644 --- a/versions.tf +++ b/versions.tf @@ -14,5 +14,9 @@ terraform { source = "hashicorp/kubernetes" version = ">= 2.10" } + time = { + source = "hashicorp/time" + version = ">= 0.9" + } } } From 60cc6d7a14a661a6b1b6b838c42cc057ce6f87aa Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 17 Feb 2023 12:28:32 +0000 Subject: [PATCH 659/962] chore(release): version 19.9.0 [skip ci] ## [19.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.8.0...v19.9.0) (2023-02-17) ### Features * Add support for enabling addons before data plane compute is created ([#2478](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2478)) ([78027f3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/78027f37e43c79748cd7528d3803122cb8072ed7)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b5b79a70b..a5d4fa750c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.8.0...v19.9.0) (2023-02-17) + + +### Features + +* Add support for enabling addons before data plane compute is created ([#2478](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2478)) ([78027f3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/78027f37e43c79748cd7528d3803122cb8072ed7)) + ## [19.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.7.0...v19.8.0) (2023-02-15) From 274c1a9222f7b73e10eaac09a8fc9282727b51be Mon Sep 17 00:00:00 2001 From: coliss86 Date: Fri, 17 Feb 2023 14:00:31 +0100 Subject: [PATCH 660/962] docs: Correct indentation for the bullet list on upgrade guide (#2479) Co-authored-by: Bryant Biggs --- docs/UPGRADE-19.0.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/UPGRADE-19.0.md b/docs/UPGRADE-19.0.md index dbdd9e6efb..99a6d40f3c 100644 --- a/docs/UPGRADE-19.0.md +++ b/docs/UPGRADE-19.0.md @@ -57,10 +57,9 @@ Please consult the `examples` directory for reference example configurations. If ### Variable and output changes 1. Removed variables: - - - `node_security_group_ntp_ipv4_cidr_block` - default security group settings have an egress rule for ALL to `0.0.0.0/0`/`::/0` - - `node_security_group_ntp_ipv6_cidr_block` - default security group settings have an egress rule for ALL to `0.0.0.0/0`/`::/0` - + + - `node_security_group_ntp_ipv4_cidr_block` - default security group settings have an egress rule for ALL to `0.0.0.0/0`/`::/0` + - `node_security_group_ntp_ipv6_cidr_block` - default security group settings have an egress rule for ALL to `0.0.0.0/0`/`::/0` - Self-managed node groups: - `create_security_group` - `security_group_name` From 8954ff7bb433358ba99b77248e3aae377d3a580b Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 17 Feb 2023 09:23:58 -0600 Subject: [PATCH 661/962] feat: Allow setting custom IRSA policy name for karpenter (#2480) --- modules/karpenter/README.md | 1 + modules/karpenter/main.tf | 7 ++++--- modules/karpenter/variables.tf | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index dfe0670844..055b6de7e6 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -163,6 +163,7 @@ No modules. | [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | | [irsa\_path](#input\_irsa\_path) | Path of IAM role for service accounts | `string` | `"/"` | no | | [irsa\_permissions\_boundary\_arn](#input\_irsa\_permissions\_boundary\_arn) | Permissions boundary ARN to use for IAM role for service accounts | `string` | `null` | no | +| [irsa\_policy\_name](#input\_irsa\_policy\_name) | Name of IAM policy for service accounts | `string` | `null` | no | | [irsa\_ssm\_parameter\_arns](#input\_irsa\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter | `list(string)` |
[
"arn:aws:ssm:*:*:parameter/aws/service/*"
]
| no | | [irsa\_subnet\_account\_id](#input\_irsa\_subnet\_account\_id) | Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account | `string` | `""` | no | | [irsa\_tag\_key](#input\_irsa\_tag\_key) | Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner | `string` | `"karpenter.sh/discovery"` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 343e38b256..f7d9f3e01f 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -13,8 +13,9 @@ locals { ################################################################################ locals { - create_irsa = var.create && var.create_irsa - irsa_name = coalesce(var.irsa_name, "KarpenterIRSA-${var.cluster_name}") + create_irsa = var.create && var.create_irsa + irsa_name = coalesce(var.irsa_name, "KarpenterIRSA-${var.cluster_name}") + irsa_policy_name = coalesce(var.irsa_policy_name, local.irsa_name) irsa_oidc_provider_url = replace(var.irsa_oidc_provider_arn, "/^(.*provider/)/", "") } @@ -159,7 +160,7 @@ data "aws_iam_policy_document" "irsa" { resource "aws_iam_policy" "irsa" { count = local.create_irsa ? 1 : 0 - name_prefix = "${local.irsa_name}-" + name_prefix = "${local.irsa_policy_name}-" path = var.irsa_path description = var.irsa_description policy = data.aws_iam_policy_document.irsa[0].json diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 8fdefc2705..df15e7ca14 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -32,6 +32,12 @@ variable "irsa_name" { default = null } +variable "irsa_policy_name" { + description = "Name of IAM policy for service accounts" + type = string + default = null +} + variable "irsa_use_name_prefix" { description = "Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix" type = bool From d7016cf39d25245eb0c2992971d711fbe8ddf71e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 17 Feb 2023 15:24:27 +0000 Subject: [PATCH 662/962] chore(release): version 19.10.0 [skip ci] ## [19.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.9.0...v19.10.0) (2023-02-17) ### Features * Allow setting custom IRSA policy name for karpenter ([#2480](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2480)) ([8954ff7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8954ff7bb433358ba99b77248e3aae377d3a580b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5d4fa750c..df43300785 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.9.0...v19.10.0) (2023-02-17) + + +### Features + +* Allow setting custom IRSA policy name for karpenter ([#2480](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2480)) ([8954ff7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8954ff7bb433358ba99b77248e3aae377d3a580b)) + ## [19.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.8.0...v19.9.0) (2023-02-17) From c7565e265e23cbc622041fc153193f490cfe948f Mon Sep 17 00:00:00 2001 From: buettner123 Date: Wed, 22 Feb 2023 14:11:16 +0100 Subject: [PATCH 663/962] docs: Add info about new variable to incompatible changes (#2489) --- docs/UPGRADE-19.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/UPGRADE-19.0.md b/docs/UPGRADE-19.0.md index 99a6d40f3c..19bd5f3946 100644 --- a/docs/UPGRADE-19.0.md +++ b/docs/UPGRADE-19.0.md @@ -12,6 +12,7 @@ Please consult the `examples` directory for reference example configurations. If 1. `var.iam_role_additional_policies` was changed from type `list(string)` to type `map(string)` -> this is a breaking change. More information on managing this change can be found below, under `Terraform State Moves` 2. The logic used in the root module for this variable was changed to replace the use of `try()` with `lookup()`. More details on why can be found [here](https://github.com/clowdhaus/terraform-for-each-unknown) - The cluster name has been removed from the Karpenter module event rule names. Due to the use of long cluster names appending to the provided naming scheme, the cluster name has moved to a `ClusterName` tag and the event rule name is now a prefix. This guarantees that users can have multiple instances of Karpenter with their respective event rules/SQS queue without name collisions, while also still being able to identify which queues and event rules belong to which cluster. +- The new variable `node_security_group_enable_recommended_rules` is set to true by default and may conflict with any custom ingress/egress rules. Please ensure that any duplicates from the `node_security_group_additional_rules` are removed before upgrading, or set `node_security_group_enable_recommended_rules` to false. [Reference](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-19.0.md#added) ## Additional changes From e257dafe94e11384caf210d9ff21c4d3e078cb17 Mon Sep 17 00:00:00 2001 From: Apoorva Kulkarni <4845260+askulkarni2@users.noreply.github.com> Date: Fri, 17 Mar 2023 06:49:08 -0700 Subject: [PATCH 664/962] fix: Return correct status for mng (#2524) --- modules/eks-managed-node-group/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index 58dbe9646f..0475953452 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -48,7 +48,7 @@ output "node_group_autoscaling_group_names" { output "node_group_status" { description = "Status of the EKS Node Group" - value = try(aws_eks_node_group.this[0].arn, null) + value = try(aws_eks_node_group.this[0].status, null) } output "node_group_labels" { From df1b62548c1d8c7117f4ab45c5b494de64b34cb8 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 17 Mar 2023 13:49:46 +0000 Subject: [PATCH 665/962] chore(release): version 19.10.1 [skip ci] ### [19.10.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.0...v19.10.1) (2023-03-17) ### Bug Fixes * Return correct status for mng ([#2524](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2524)) ([e257daf](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e257dafe94e11384caf210d9ff21c4d3e078cb17)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df43300785..611cdb1af2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.10.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.0...v19.10.1) (2023-03-17) + + +### Bug Fixes + +* Return correct status for mng ([#2524](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2524)) ([e257daf](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e257dafe94e11384caf210d9ff21c4d3e078cb17)) + ## [19.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.9.0...v19.10.0) (2023-02-17) From 90bba6e9d8b13d228ae82471b715c9d16ced5391 Mon Sep 17 00:00:00 2001 From: Mossaab Stiri Date: Thu, 23 Mar 2023 16:36:26 +0100 Subject: [PATCH 666/962] docs: Update Documentation README ToC (#2532) --- docs/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/README.md b/docs/README.md index 3806b326e5..889b8481e4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,3 +10,4 @@ - Upgrade Guides - [Upgrade to v17.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-17.0.md) - [Upgrade to v18.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-18.0.md) + - [Upgrade to v19.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-19.0.md) From e64a490d8db4ebf495f42c542a40d7d763005873 Mon Sep 17 00:00:00 2001 From: visokoo Date: Thu, 23 Mar 2023 08:56:42 -0700 Subject: [PATCH 667/962] fix: Add Name tag for EKS cloudwatch log group (#2500) Co-authored-by: Vivian Ta Co-authored-by: Bryant Biggs --- main.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 604f90ce86..52d36aa201 100644 --- a/main.tf +++ b/main.tf @@ -110,7 +110,10 @@ resource "aws_cloudwatch_log_group" "this" { retention_in_days = var.cloudwatch_log_group_retention_in_days kms_key_id = var.cloudwatch_log_group_kms_key_id - tags = var.tags + tags = merge( + var.tags, + { Name = "/aws/eks/${var.cluster_name}/cluster" } + ) } ################################################################################ From 1c878618a0f2b8764e2c12639db3155e3b79f5ca Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 23 Mar 2023 15:57:15 +0000 Subject: [PATCH 668/962] chore(release): version 19.10.2 [skip ci] ### [19.10.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.1...v19.10.2) (2023-03-23) ### Bug Fixes * Add Name tag for EKS cloudwatch log group ([#2500](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2500)) ([e64a490](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e64a490d8db4ebf495f42c542a40d7d763005873)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 611cdb1af2..ac3cd1a0fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.10.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.1...v19.10.2) (2023-03-23) + + +### Bug Fixes + +* Add Name tag for EKS cloudwatch log group ([#2500](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2500)) ([e64a490](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e64a490d8db4ebf495f42c542a40d7d763005873)) + ### [19.10.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.0...v19.10.1) (2023-03-17) From f977d83500ac529b09918d4e78aa8887749a8cd1 Mon Sep 17 00:00:00 2001 From: Cody Grant Date: Thu, 23 Mar 2023 12:03:02 -0400 Subject: [PATCH 669/962] fix: Add `aws_eks_addons.before_compute` to the `cluster_addons` output (#2533) Co-authored-by: Bryant Biggs --- outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/outputs.tf b/outputs.tf index f0cb797e14..ea02a3a8cc 100644 --- a/outputs.tf +++ b/outputs.tf @@ -143,7 +143,7 @@ output "cluster_iam_role_unique_id" { output "cluster_addons" { description = "Map of attribute maps for all EKS cluster addons enabled" - value = aws_eks_addon.this + value = merge(aws_eks_addon.this, aws_eks_addon.before_compute) } ################################################################################ From c013f7bfbc9060f8463987cd12dbca9adfd4049d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 23 Mar 2023 16:03:35 +0000 Subject: [PATCH 670/962] chore(release): version 19.10.3 [skip ci] ### [19.10.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.2...v19.10.3) (2023-03-23) ### Bug Fixes * Add `aws_eks_addons.before_compute` to the `cluster_addons` output ([#2533](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2533)) ([f977d83](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f977d83500ac529b09918d4e78aa8887749a8cd1)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac3cd1a0fd..78f75ad921 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.10.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.2...v19.10.3) (2023-03-23) + + +### Bug Fixes + +* Add `aws_eks_addons.before_compute` to the `cluster_addons` output ([#2533](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2533)) ([f977d83](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f977d83500ac529b09918d4e78aa8887749a8cd1)) + ### [19.10.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.1...v19.10.2) (2023-03-23) From bd387d69fac5a431a426e12de786ab80aea112a6 Mon Sep 17 00:00:00 2001 From: Constantin Bugneac <11664677+Constantin07@users.noreply.github.com> Date: Tue, 28 Mar 2023 21:50:40 +0100 Subject: [PATCH 671/962] feat: Add optional list of policy ARNs for attachment to Karpenter IRSA (#2537) Co-authored-by: Bryant Biggs --- examples/karpenter/main.tf | 56 ++++++++++++++++++++++------------ modules/karpenter/README.md | 2 ++ modules/karpenter/main.tf | 7 +++++ modules/karpenter/variables.tf | 6 ++++ 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 497fab71ac..4e64b9bb61 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -84,6 +84,26 @@ module "eks" { coredns = { configuration_values = jsonencode({ computeType = "Fargate" + # Ensure that the we fully utilize the minimum amount of resources that are supplied by + # Fargate https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html + # Fargate adds 256 MB to each pod's memory reservation for the required Kubernetes + # components (kubelet, kube-proxy, and containerd). Fargate rounds up to the following + # compute configuration that most closely matches the sum of vCPU and memory requests in + # order to ensure pods always have the resources that they need to run. + resources = { + limits = { + cpu = "0.25" + # We are targetting the smallest Task size of 512Mb, so we subtract 256Mb from the + # request/limit to ensure we can fit within that task + memory = "256M" + } + requests = { + cpu = "0.25" + # We are targetting the smallest Task size of 512Mb, so we subtract 256Mb from the + # request/limit to ensure we can fit within that task + memory = "256M" + } + } }) } } @@ -109,26 +129,18 @@ module "eks" { }, ] - fargate_profiles = merge( - { for i in range(3) : - "kube-system-${element(split("-", local.azs[i]), 2)}" => { - selectors = [ - { namespace = "kube-system" } - ] - # We want to create a profile per AZ for high availability - subnet_ids = [element(module.vpc.private_subnets, i)] - } - }, - { for i in range(3) : - "karpenter-${element(split("-", local.azs[i]), 2)}" => { - selectors = [ - { namespace = "karpenter" } - ] - # We want to create a profile per AZ for high availability - subnet_ids = [element(module.vpc.private_subnets, i)] - } - }, - ) + fargate_profiles = { + karpenter = { + selectors = [ + { namespace = "karpenter" } + ] + } + kube-system = { + selectors = [ + { namespace = "kube-system" } + ] + } + } tags = merge(local.tags, { # NOTE - if creating multiple security groups with this module, only tag the @@ -148,6 +160,10 @@ module "karpenter" { cluster_name = module.eks.cluster_name irsa_oidc_provider_arn = module.eks.oidc_provider_arn + policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + tags = local.tags } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 055b6de7e6..89f5495f33 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -124,6 +124,7 @@ No modules. | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.irsa_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_sqs_queue.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | | [aws_sqs_queue_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | @@ -169,6 +170,7 @@ No modules. | [irsa\_tag\_key](#input\_irsa\_tag\_key) | Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner | `string` | `"karpenter.sh/discovery"` | no | | [irsa\_tags](#input\_irsa\_tags) | A map of additional tags to add the the IAM role for service accounts | `map(any)` | `{}` | no | | [irsa\_use\_name\_prefix](#input\_irsa\_use\_name\_prefix) | Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix | `bool` | `true` | no | +| [policies](#input\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | | [queue\_kms\_data\_key\_reuse\_period\_seconds](#input\_queue\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again | `number` | `null` | no | | [queue\_kms\_master\_key\_id](#input\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no | | [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index f7d9f3e01f..cdbfb13f5d 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -175,6 +175,13 @@ resource "aws_iam_role_policy_attachment" "irsa" { policy_arn = aws_iam_policy.irsa[0].arn } +resource "aws_iam_role_policy_attachment" "irsa_additional" { + for_each = { for k, v in var.policies : k => v if local.create_irsa } + + role = aws_iam_role.irsa[0].name + policy_arn = each.value +} + ################################################################################ # Node Termination Queue ################################################################################ diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index df15e7ca14..f92160f2d4 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -74,6 +74,12 @@ variable "irsa_tags" { default = {} } +variable "policies" { + description = "Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format" + type = map(string) + default = {} +} + variable "irsa_tag_key" { description = "Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner" type = string From 2cf6a5bc14fd48b8e02c9755648b862d41e00333 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 28 Mar 2023 20:51:12 +0000 Subject: [PATCH 672/962] chore(release): version 19.11.0 [skip ci] ## [19.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.3...v19.11.0) (2023-03-28) ### Features * Add optional list of policy ARNs for attachment to Karpenter IRSA ([#2537](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2537)) ([bd387d6](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/bd387d69fac5a431a426e12de786ab80aea112a6)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78f75ad921..eba78413fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.3...v19.11.0) (2023-03-28) + + +### Features + +* Add optional list of policy ARNs for attachment to Karpenter IRSA ([#2537](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2537)) ([bd387d6](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/bd387d69fac5a431a426e12de786ab80aea112a6)) + ### [19.10.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.2...v19.10.3) (2023-03-23) From 4a2523cddd4498f3ece5aee2eedf618dd701eb59 Mon Sep 17 00:00:00 2001 From: Le Minh Duc Date: Fri, 31 Mar 2023 21:46:27 +0700 Subject: [PATCH 673/962] feat: Add Autoscaling schedule for EKS managed node group (#2504) Co-authored-by: Bryant Biggs --- examples/eks_managed_node_group/main.tf | 21 ++++++++++++++++++++ modules/eks-managed-node-group/README.md | 4 ++++ modules/eks-managed-node-group/main.tf | 22 +++++++++++++++++++++ modules/eks-managed-node-group/outputs.tf | 9 +++++++++ modules/eks-managed-node-group/variables.tf | 16 +++++++++++++++ modules/self-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/outputs.tf | 2 +- node_groups.tf | 5 ++++- 8 files changed, 78 insertions(+), 3 deletions(-) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 75210c00ec..c28e9af344 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -264,6 +264,27 @@ module "eks" { additional = aws_iam_policy.node_additional.arn } + schedules = { + scale-up = { + min_size = 2 + max_size = "-1" # Retains current max size + desired_size = 2 + start_time = "2023-03-05T00:00:00Z" + end_time = "2024-03-05T00:00:00Z" + timezone = "Etc/GMT+0" + recurrence = "0 0 * * *" + }, + scale-down = { + min_size = 0 + max_size = "-1" # Retains current max size + desired_size = 0 + start_time = "2023-03-05T12:00:00Z" + end_time = "2024-03-05T12:00:00Z" + timezone = "Etc/GMT+0" + recurrence = "0 12 * * *" + } + } + tags = { ExtraTag = "EKS managed node group complete example" } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index d94f815075..4bf3434663 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -82,6 +82,7 @@ module "eks_managed_node_group" { | Name | Type | |------|------| +| [aws_autoscaling_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_schedule) | resource | | [aws_eks_node_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | @@ -113,6 +114,7 @@ module "eks_managed_node_group" { | [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | +| [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | | [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | | [desired\_size](#input\_desired\_size) | Desired number of instances/nodes | `number` | `1` | no | | [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | @@ -159,6 +161,7 @@ module "eks_managed_node_group" { | [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet | `map(string)` | `{}` | no | | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | | [remote\_access](#input\_remote\_access) | Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false` | `any` | `{}` | no | +| [schedules](#input\_schedules) | Map of autoscaling group schedule to create | `map(any)` | `{}` | no | | [subnet\_ids](#input\_subnet\_ids) | Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME` | `list(string)` | `null` | no | | [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | @@ -175,6 +178,7 @@ module "eks_managed_node_group" { | Name | Description | |------|-------------| +| [autoscaling\_group\_schedule\_arns](#output\_autoscaling\_group\_schedule\_arns) | ARNs of autoscaling group schedules | | [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | | [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | | [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index dba2b58b5a..1d3fe81698 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -447,3 +447,25 @@ resource "aws_iam_role_policy_attachment" "additional" { policy_arn = each.value role = aws_iam_role.this[0].name } + +################################################################################ +# Autoscaling Group Schedule +################################################################################ + +resource "aws_autoscaling_schedule" "this" { + for_each = { for k, v in var.schedules : k => v if var.create && var.create_schedule } + + scheduled_action_name = each.key + autoscaling_group_name = aws_eks_node_group.this[0].resources[0].autoscaling_groups[0].name + + min_size = try(each.value.min_size, null) + max_size = try(each.value.max_size, null) + desired_capacity = try(each.value.desired_size, null) + start_time = try(each.value.start_time, null) + end_time = try(each.value.end_time, null) + time_zone = try(each.value.time_zone, null) + + # [Minute] [Hour] [Day_of_Month] [Month_of_Year] [Day_of_Week] + # Cron examples: https://crontab.guru/examples.html + recurrence = try(each.value.recurrence, null) +} diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index 0475953452..012cd46f4e 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -61,6 +61,15 @@ output "node_group_taints" { value = try(aws_eks_node_group.this[0].taint, []) } +################################################################################ +# Autoscaling Group Schedule +################################################################################ + +output "autoscaling_group_schedule_arns" { + description = "ARNs of autoscaling group schedules" + value = { for k, v in aws_autoscaling_schedule.this : k => v.arn } +} + ################################################################################ # IAM Role ################################################################################ diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 36645e0d90..197cd28c95 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -469,3 +469,19 @@ variable "iam_role_tags" { type = map(string) default = {} } + +################################################################################ +# Autoscaling Group Schedule +################################################################################ + +variable "create_schedule" { + description = "Determines whether to create autoscaling group schedule or not" + type = bool + default = true +} + +variable "schedules" { + description = "Map of autoscaling group schedule to create" + type = map(any) + default = {} +} diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 6ec1534bd1..9b72f49a9f 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -680,7 +680,7 @@ resource "aws_autoscaling_group" "this" { ################################################################################ resource "aws_autoscaling_schedule" "this" { - for_each = var.create && var.create_schedule ? var.schedules : {} + for_each = { for k, v in var.schedules : k => v if var.create && var.create_schedule } scheduled_action_name = each.key autoscaling_group_name = aws_autoscaling_group.this[0].name diff --git a/modules/self-managed-node-group/outputs.tf b/modules/self-managed-node-group/outputs.tf index c9c9b75254..5c83497218 100644 --- a/modules/self-managed-node-group/outputs.tf +++ b/modules/self-managed-node-group/outputs.tf @@ -82,7 +82,7 @@ output "autoscaling_group_vpc_zone_identifier" { } ################################################################################ -# autoscaling group schedule +# Autoscaling Group Schedule ################################################################################ output "autoscaling_group_schedule_arns" { diff --git a/node_groups.tf b/node_groups.tf index 0dacc1ed23..db78861a81 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -363,6 +363,9 @@ module "eks_managed_node_group" { # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.eks_managed_node_group_defaults, "iam_role_additional_policies", {})) + create_schedule = try(each.value.create_schedule, var.eks_managed_node_group_defaults.create_schedule, true) + schedules = try(each.value.schedules, var.eks_managed_node_group_defaults.schedules, {}) + # Security group vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.eks_managed_node_group_defaults.vpc_security_group_ids, []))) cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, var.eks_managed_node_group_defaults.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null @@ -426,7 +429,7 @@ module "self_managed_node_group" { mixed_instances_policy = try(each.value.mixed_instances_policy, var.self_managed_node_group_defaults.mixed_instances_policy, null) warm_pool = try(each.value.warm_pool, var.self_managed_node_group_defaults.warm_pool, {}) - create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, false) + create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, true) schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, {}) delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) From 273c48a631c0ab45ffac1d1a77c7752e8541b1f2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 31 Mar 2023 14:46:55 +0000 Subject: [PATCH 674/962] chore(release): version 19.12.0 [skip ci] ## [19.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.11.0...v19.12.0) (2023-03-31) ### Features * Add Autoscaling schedule for EKS managed node group ([#2504](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2504)) ([4a2523c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/4a2523cddd4498f3ece5aee2eedf618dd701eb59)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eba78413fa..c3cf7d9ed5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.11.0...v19.12.0) (2023-03-31) + + +### Features + +* Add Autoscaling schedule for EKS managed node group ([#2504](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2504)) ([4a2523c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/4a2523cddd4498f3ece5aee2eedf618dd701eb59)) + ## [19.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.10.3...v19.11.0) (2023-03-28) From 54417d244c06b459b399e84433343af6e9934bb3 Mon Sep 17 00:00:00 2001 From: petros-d <46783518+petros-d@users.noreply.github.com> Date: Wed, 12 Apr 2023 23:05:33 +0100 Subject: [PATCH 675/962] feat: Add support for allowed_instance_type (#2552) --- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/main.tf | 16 +++++++++++++--- modules/self-managed-node-group/versions.tf | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index f505e09fd3..a9eff4772e 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | ## Modules diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 9b72f49a9f..049e32dfca 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -197,8 +197,9 @@ resource "aws_launch_template" "this" { } } - accelerator_types = try(instance_requirements.value.accelerator_types, []) - bare_metal = try(instance_requirements.value.bare_metal, null) + accelerator_types = try(instance_requirements.value.accelerator_types, []) + allowed_instance_types = try(instance_requirements.value.allowed_instance_types, null) + bare_metal = try(instance_requirements.value.bare_metal, null) dynamic "baseline_ebs_bandwidth_mbps" { for_each = try([instance_requirements.value.baseline_ebs_bandwidth_mbps], []) @@ -211,7 +212,7 @@ resource "aws_launch_template" "this" { burstable_performance = try(instance_requirements.value.burstable_performance, null) cpu_manufacturers = try(instance_requirements.value.cpu_manufacturers, []) - excluded_instance_types = try(instance_requirements.value.excluded_instance_types, []) + excluded_instance_types = try(instance_requirements.value.excluded_instance_types, null) instance_generations = try(instance_requirements.value.instance_generations, []) local_storage = try(instance_requirements.value.local_storage, null) local_storage_types = try(instance_requirements.value.local_storage_types, []) @@ -234,6 +235,15 @@ resource "aws_launch_template" "this" { } } + dynamic "network_bandwidth_gbps" { + for_each = try([instance_requirements.value.network_bandwidth_gbps], []) + + content { + max = try(network_bandwidth_gbps.value.max, null) + min = try(network_bandwidth_gbps.value.min, null) + } + } + dynamic "network_interface_count" { for_each = try([instance_requirements.value.network_interface_count], []) diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 55eff62b09..01d187af62 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } } } From 6770bf177e49acda87de7f3ff16327e20bd0ac54 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 12 Apr 2023 22:06:08 +0000 Subject: [PATCH 676/962] chore(release): version 19.13.0 [skip ci] ## [19.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.12.0...v19.13.0) (2023-04-12) ### Features * Add support for allowed_instance_type ([#2552](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2552)) ([54417d2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/54417d244c06b459b399e84433343af6e9934bb3)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3cf7d9ed5..177d1b4365 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.12.0...v19.13.0) (2023-04-12) + + +### Features + +* Add support for allowed_instance_type ([#2552](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2552)) ([54417d2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/54417d244c06b459b399e84433343af6e9934bb3)) + ## [19.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.11.0...v19.12.0) (2023-03-31) From fd62c24b07359082985b8c7422361afdeec4254c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9C=A0=EC=9A=A9=ED=99=98?= <33824408+yoosful@users.noreply.github.com> Date: Thu, 13 Apr 2023 07:07:49 +0900 Subject: [PATCH 677/962] docs: Minor fix in comment (#2547) Co-authored-by: Bryant Biggs --- examples/karpenter/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 4e64b9bb61..ac2ef4870e 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -84,7 +84,7 @@ module "eks" { coredns = { configuration_values = jsonencode({ computeType = "Fargate" - # Ensure that the we fully utilize the minimum amount of resources that are supplied by + # Ensure that we fully utilize the minimum amount of resources that are supplied by # Fargate https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html # Fargate adds 256 MB to each pod's memory reservation for the required Kubernetes # components (kubelet, kube-proxy, and containerd). Fargate rounds up to the following From 969c7a7c4340c8ed327d18f86c5e00e18190a48b Mon Sep 17 00:00:00 2001 From: tal-lederman-dy <125883319+tal-lederman-dy@users.noreply.github.com> Date: Tue, 18 Apr 2023 16:03:15 +0300 Subject: [PATCH 678/962] fix: SQS queue encryption types selection (#2575) Co-authored-by: Tal Lederman --- modules/karpenter/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index cdbfb13f5d..fdae1a410a 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -197,7 +197,7 @@ resource "aws_sqs_queue" "this" { name = local.queue_name message_retention_seconds = 300 - sqs_managed_sse_enabled = var.queue_managed_sse_enabled + sqs_managed_sse_enabled = var.queue_managed_sse_enabled ? var.queue_managed_sse_enabled : null kms_master_key_id = var.queue_kms_master_key_id kms_data_key_reuse_period_seconds = var.queue_kms_data_key_reuse_period_seconds From 0f9d9fac93caf239386f47b0f117706ea78c2bec Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 18 Apr 2023 13:03:54 +0000 Subject: [PATCH 679/962] chore(release): version 19.13.1 [skip ci] ### [19.13.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.13.0...v19.13.1) (2023-04-18) ### Bug Fixes * SQS queue encryption types selection ([#2575](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2575)) ([969c7a7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/969c7a7c4340c8ed327d18f86c5e00e18190a48b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 177d1b4365..0603aa0452 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.13.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.13.0...v19.13.1) (2023-04-18) + + +### Bug Fixes + +* SQS queue encryption types selection ([#2575](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2575)) ([969c7a7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/969c7a7c4340c8ed327d18f86c5e00e18190a48b)) + ## [19.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.12.0...v19.13.0) (2023-04-12) From aa3bdf1c19747bca7067c6e49c071ae80a9ca5e5 Mon Sep 17 00:00:00 2001 From: Tung Nguyen Date: Wed, 17 May 2023 07:04:11 -0700 Subject: [PATCH 680/962] feat: Add irsa_tag_values variable (#2584) --- modules/karpenter/README.md | 1 + modules/karpenter/main.tf | 8 ++++++-- modules/karpenter/variables.tf | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 89f5495f33..e1c1abd92a 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -168,6 +168,7 @@ No modules. | [irsa\_ssm\_parameter\_arns](#input\_irsa\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter | `list(string)` |
[
"arn:aws:ssm:*:*:parameter/aws/service/*"
]
| no | | [irsa\_subnet\_account\_id](#input\_irsa\_subnet\_account\_id) | Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account | `string` | `""` | no | | [irsa\_tag\_key](#input\_irsa\_tag\_key) | Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner | `string` | `"karpenter.sh/discovery"` | no | +| [irsa\_tag\_values](#input\_irsa\_tag\_values) | Tag values (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner. Defaults to cluster name when not set. | `list(string)` | `null` | no | | [irsa\_tags](#input\_irsa\_tags) | A map of additional tags to add the the IAM role for service accounts | `map(any)` | `{}` | no | | [irsa\_use\_name\_prefix](#input\_irsa\_use\_name\_prefix) | Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix | `bool` | `true` | no | | [policies](#input\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index fdae1a410a..2c809ed621 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -63,6 +63,10 @@ resource "aws_iam_role" "irsa" { tags = merge(var.tags, var.irsa_tags) } +locals { + irsa_tag_values = coalescelist([var.cluster_name], var.irsa_tag_values) +} + data "aws_iam_policy_document" "irsa" { count = local.create_irsa ? 1 : 0 @@ -97,7 +101,7 @@ data "aws_iam_policy_document" "irsa" { condition { test = "StringEquals" variable = "ec2:ResourceTag/${var.irsa_tag_key}" - values = [var.cluster_name] + values = local.irsa_tag_values } } @@ -110,7 +114,7 @@ data "aws_iam_policy_document" "irsa" { condition { test = "StringEquals" variable = "ec2:ResourceTag/${var.irsa_tag_key}" - values = [var.cluster_name] + values = local.irsa_tag_values } } diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index f92160f2d4..47c42bace9 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -86,6 +86,12 @@ variable "irsa_tag_key" { default = "karpenter.sh/discovery" } +variable "irsa_tag_values" { + description = "Tag values (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner. Defaults to cluster name when not set." + type = list(string) + default = null +} + variable "irsa_ssm_parameter_arns" { description = "List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter" type = list(string) From 36fc3b08e4ba8f7759ca8b39b71783df11b90915 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 17 May 2023 14:04:41 +0000 Subject: [PATCH 681/962] chore(release): version 19.14.0 [skip ci] ## [19.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.13.1...v19.14.0) (2023-05-17) ### Features * Add irsa_tag_values variable ([#2584](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2584)) ([aa3bdf1](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/aa3bdf1c19747bca7067c6e49c071ae80a9ca5e5)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0603aa0452..f5b35cc8cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.13.1...v19.14.0) (2023-05-17) + + +### Features + +* Add irsa_tag_values variable ([#2584](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2584)) ([aa3bdf1](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/aa3bdf1c19747bca7067c6e49c071ae80a9ca5e5)) + ### [19.13.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.13.0...v19.13.1) (2023-04-18) From 703a76bf8903a0f37d369bd88e4e8694df06128b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=B8=ED=99=98?= Date: Wed, 17 May 2023 23:13:02 +0900 Subject: [PATCH 682/962] docs: Add more description in cluster_encryption_config input variable (#2609) Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 2 +- README.md | 2 +- variables.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 071427d757..4395e67e7f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.1 + rev: v1.79.1 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/README.md b/README.md index f265acf94a..9773afdbe3 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no | | [cluster\_addons\_timeouts](#input\_cluster\_addons\_timeouts) | Create, update, and delete timeout configurations for the cluster addons | `map(string)` | `{}` | no | | [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | -| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster | `any` |
{
"resources": [
"secrets"
]
}
| no | +| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster. To disable secret encryption, set this value to `{}` | `any` |
{
"resources": [
"secrets"
]
}
| no | | [cluster\_encryption\_policy\_description](#input\_cluster\_encryption\_policy\_description) | Description of the cluster encryption policy created | `string` | `"Cluster encryption policy to allow cluster role to utilize CMK provided"` | no | | [cluster\_encryption\_policy\_name](#input\_cluster\_encryption\_policy\_name) | Name to use on cluster encryption policy created | `string` | `null` | no | | [cluster\_encryption\_policy\_path](#input\_cluster\_encryption\_policy\_path) | Cluster encryption policy path | `string` | `null` | no | diff --git a/variables.tf b/variables.tf index dc96661810..b8577737d8 100644 --- a/variables.tf +++ b/variables.tf @@ -99,7 +99,7 @@ variable "outpost_config" { } variable "cluster_encryption_config" { - description = "Configuration block with encryption configuration for the cluster" + description = "Configuration block with encryption configuration for the cluster. To disable secret encryption, set this value to `{}`" type = any default = { resources = ["secrets"] From fcb4256fa17b3e0bde8bbd69dfd191414977f30c Mon Sep 17 00:00:00 2001 From: tculp Date: Wed, 17 May 2023 10:22:26 -0400 Subject: [PATCH 683/962] docs: Re-add 'Tags for the ASG to support cluster-autoscaler scale up from 0' example (#2494) Co-authored-by: Bryant Biggs --- examples/eks_managed_node_group/README.md | 1 + examples/eks_managed_node_group/main.tf | 49 +++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 9328a05e5b..ca0594b06e 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -53,6 +53,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| +| [aws_autoscaling_group_tag.cluster_autoscaler_label_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group_tag) | resource | | [aws_iam_policy.node_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_security_group.remote_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index c28e9af344..beaba19de2 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -462,3 +462,52 @@ data "aws_ami" "eks_default_bottlerocket" { values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"] } } + +################################################################################ +# Tags for the ASG to support cluster-autoscaler scale up from 0 +################################################################################ + +locals { + + # We need to lookup K8s taint effect from the AWS API value + taint_effects = { + NO_SCHEDULE = "NoSchedule" + NO_EXECUTE = "NoExecute" + PREFER_NO_SCHEDULE = "PreferNoSchedule" + } + + cluster_autoscaler_label_tags = merge([ + for name, group in module.eks.eks_managed_node_groups : { + for label_name, label_value in coalesce(group.node_group_labels, {}) : "${name}|label|${label_name}" => { + autoscaling_group = group.node_group_autoscaling_group_names[0], + key = "k8s.io/cluster-autoscaler/node-template/label/${label_name}", + value = label_value, + } + } + ]...) + + cluster_autoscaler_taint_tags = merge([ + for name, group in module.eks.eks_managed_node_groups : { + for taint in coalesce(group.node_group_taints, []) : "${name}|taint|${taint.key}" => { + autoscaling_group = group.node_group_autoscaling_group_names[0], + key = "k8s.io/cluster-autoscaler/node-template/taint/${taint.key}" + value = "${taint.value}:${local.taint_effects[taint.effect]}" + } + } + ]...) + + cluster_autoscaler_asg_tags = merge(local.cluster_autoscaler_label_tags, local.cluster_autoscaler_taint_tags) +} + +resource "aws_autoscaling_group_tag" "cluster_autoscaler_label_tags" { + for_each = local.cluster_autoscaler_asg_tags + + autoscaling_group_name = each.value.autoscaling_group + + tag { + key = each.value.key + value = each.value.value + + propagate_at_launch = false + } +} From f8ea3d08adbc4abfb18a77ad44e30b93cd05c050 Mon Sep 17 00:00:00 2001 From: Eric Bailey Date: Wed, 24 May 2023 12:46:52 -0500 Subject: [PATCH 684/962] feat: Ignore changes to *.aws_iam_role.*.role_last_used (#2628) --- main.tf | 6 ++++++ modules/eks-managed-node-group/main.tf | 6 ++++++ modules/karpenter/main.tf | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/main.tf b/main.tf index 52d36aa201..7fad0019fa 100644 --- a/main.tf +++ b/main.tf @@ -318,6 +318,12 @@ resource "aws_iam_role" "this" { } tags = merge(var.tags, var.iam_role_tags) + + lifecycle { + ignore_changes = [ + role_last_used, + ] + } } # Policies attached ref https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 1d3fe81698..3deca4dcea 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -427,6 +427,12 @@ resource "aws_iam_role" "this" { force_detach_policies = true tags = merge(var.tags, var.iam_role_tags) + + lifecycle { + ignore_changes = [ + role_last_used, + ] + } } # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 2c809ed621..edfa556e08 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -61,6 +61,12 @@ resource "aws_iam_role" "irsa" { force_detach_policies = true tags = merge(var.tags, var.irsa_tags) + + lifecycle { + ignore_changes = [ + role_last_used, + ] + } } locals { From b168c1bf41b2aeaa2741738ca59ce06ad66fadab Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 24 May 2023 17:47:20 +0000 Subject: [PATCH 685/962] chore(release): version 19.15.0 [skip ci] ## [19.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.14.0...v19.15.0) (2023-05-24) ### Features * Ignore changes to *.aws_iam_role.*.role_last_used ([#2628](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2628)) ([f8ea3d0](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f8ea3d08adbc4abfb18a77ad44e30b93cd05c050)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5b35cc8cb..c9178e5e7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.14.0...v19.15.0) (2023-05-24) + + +### Features + +* Ignore changes to *.aws_iam_role.*.role_last_used ([#2628](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2628)) ([f8ea3d0](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f8ea3d08adbc4abfb18a77ad44e30b93cd05c050)) + ## [19.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.13.1...v19.14.0) (2023-05-17) From e23139ad2da0c31c8aa644ae0516ba9ee2a66399 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 24 May 2023 17:54:33 -0400 Subject: [PATCH 686/962] fix: Revert changes to ignore `role_last_used` (#2629) --- main.tf | 6 ------ modules/eks-managed-node-group/main.tf | 6 ------ modules/karpenter/main.tf | 6 ------ 3 files changed, 18 deletions(-) diff --git a/main.tf b/main.tf index 7fad0019fa..52d36aa201 100644 --- a/main.tf +++ b/main.tf @@ -318,12 +318,6 @@ resource "aws_iam_role" "this" { } tags = merge(var.tags, var.iam_role_tags) - - lifecycle { - ignore_changes = [ - role_last_used, - ] - } } # Policies attached ref https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 3deca4dcea..1d3fe81698 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -427,12 +427,6 @@ resource "aws_iam_role" "this" { force_detach_policies = true tags = merge(var.tags, var.iam_role_tags) - - lifecycle { - ignore_changes = [ - role_last_used, - ] - } } # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index edfa556e08..2c809ed621 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -61,12 +61,6 @@ resource "aws_iam_role" "irsa" { force_detach_policies = true tags = merge(var.tags, var.irsa_tags) - - lifecycle { - ignore_changes = [ - role_last_used, - ] - } } locals { From f741db1b2c7c4ef0409d7c2e4f588bbc639cbaf2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 24 May 2023 21:55:10 +0000 Subject: [PATCH 687/962] chore(release): version 19.15.1 [skip ci] ### [19.15.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.0...v19.15.1) (2023-05-24) ### Bug Fixes * Revert changes to ignore `role_last_used` ([#2629](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2629)) ([e23139a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e23139ad2da0c31c8aa644ae0516ba9ee2a66399)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9178e5e7b..c933d4fe2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.15.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.0...v19.15.1) (2023-05-24) + + +### Bug Fixes + +* Revert changes to ignore `role_last_used` ([#2629](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2629)) ([e23139a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e23139ad2da0c31c8aa644ae0516ba9ee2a66399)) + ## [19.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.14.0...v19.15.0) (2023-05-24) From 6c56e2ad20057a5672526b5484df96806598a4e2 Mon Sep 17 00:00:00 2001 From: Joshua Pare <65102852+sbhrule15@users.noreply.github.com> Date: Tue, 30 May 2023 17:55:40 -0500 Subject: [PATCH 688/962] fix: Ensure `isra_tag_values` can be tried before defaulting to `cluster_name` on Karpenter module (#2631) Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 2 +- README.md | 4 +-- docs/compute_resources.md | 6 ++--- docs/irsa_integration.md | 2 +- examples/complete/README.md | 4 +-- examples/complete/main.tf | 13 +++------- examples/eks_managed_node_group/README.md | 2 +- examples/eks_managed_node_group/main.tf | 30 ++++++++++------------ examples/fargate_profile/README.md | 2 +- examples/fargate_profile/main.tf | 13 +++------- examples/karpenter/README.md | 2 +- examples/karpenter/main.tf | 13 +++------- examples/outposts/main.tf | 2 +- examples/outposts/prerequisites/main.tf | 7 +++-- examples/self_managed_node_group/README.md | 2 +- examples/self_managed_node_group/main.tf | 13 +++------- modules/eks-managed-node-group/README.md | 2 +- modules/karpenter/README.md | 2 +- modules/karpenter/main.tf | 2 +- modules/karpenter/variables.tf | 2 +- modules/self-managed-node-group/README.md | 2 +- variables.tf | 2 +- 22 files changed, 52 insertions(+), 77 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4395e67e7f..dabb150840 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.79.1 + rev: v1.80.0 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/README.md b/README.md index 9773afdbe3..a87400e1c0 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ module "eks" { version = "~> 19.0" cluster_name = "my-cluster" - cluster_version = "1.24" + cluster_version = "1.27" cluster_endpoint_public_access = true @@ -318,7 +318,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_service\_ipv6\_cidr](#input\_cluster\_service\_ipv6\_cidr) | The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.24`) | `string` | `null` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.27`) | `string` | `null` | no | | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 5e1d59f03c..29fa2efb0f 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -115,9 +115,9 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 1. The `self-managed-node-group` uses the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version by default: ```hcl - cluster_version = "1.24" + cluster_version = "1.27" - # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.24 + # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.27 self_managed_node_groups = { default = {} } @@ -126,7 +126,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 2. To use Bottlerocket, specify the `platform` as `bottlerocket` and supply a Bottlerocket OS AMI: ```hcl - cluster_version = "1.24" + cluster_version = "1.27" self_managed_node_groups = { bottlerocket = { diff --git a/docs/irsa_integration.md b/docs/irsa_integration.md index cde709fd2c..cc6a549500 100644 --- a/docs/irsa_integration.md +++ b/docs/irsa_integration.md @@ -8,7 +8,7 @@ module "eks" { source = "terraform-aws-modules/eks/aws" cluster_name = "example" - cluster_version = "1.24" + cluster_version = "1.27" cluster_addons = { vpc-cni = { diff --git a/examples/complete/README.md b/examples/complete/README.md index 39b0eb3272..c03547b5e2 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -54,9 +54,9 @@ Note that this example may create resources which cost money. Run `terraform des | [eks](#module\_eks) | ../.. | n/a | | [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | | [fargate\_profile](#module\_fargate\_profile) | ../../modules/fargate-profile | n/a | -| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 1.1.0 | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | ~> 1.5 | | [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ../../modules/self-managed-node-group | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | ## Resources diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 2917c5bc37..e946906821 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -397,7 +397,7 @@ module "disabled_self_managed_node_group" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" + version = "~> 4.0" name = local.name cidr = local.vpc_cidr @@ -407,13 +407,8 @@ module "vpc" { public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - enable_flow_log = true - create_flow_log_cloudwatch_iam_role = true - create_flow_log_cloudwatch_log_group = true + enable_nat_gateway = true + single_nat_gateway = true public_subnet_tags = { "kubernetes.io/role/elb" = 1 @@ -463,7 +458,7 @@ resource "aws_iam_policy" "additional" { module "kms" { source = "terraform-aws-modules/kms/aws" - version = "1.1.0" + version = "~> 1.5" aliases = ["eks/${local.name}"] description = "${local.name} cluster encryption key" diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index ca0594b06e..090273396c 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -46,7 +46,7 @@ Note that this example may create resources which cost money. Run `terraform des | [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 1.5 | | [eks](#module\_eks) | ../.. | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | | [vpc\_cni\_irsa](#module\_vpc\_cni\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.0 | ## Resources diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index beaba19de2..1fb1b8b723 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -19,7 +19,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.24" + cluster_version = "1.27" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -300,7 +300,7 @@ module "eks" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" + version = "~> 4.0" name = local.name cidr = local.vpc_cidr @@ -310,21 +310,17 @@ module "vpc" { public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] - enable_ipv6 = true - assign_ipv6_address_on_creation = true - create_egress_only_igw = true - - public_subnet_ipv6_prefixes = [0, 1, 2] - private_subnet_ipv6_prefixes = [3, 4, 5] - intra_subnet_ipv6_prefixes = [6, 7, 8] - - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - enable_flow_log = true - create_flow_log_cloudwatch_iam_role = true - create_flow_log_cloudwatch_log_group = true + enable_nat_gateway = true + single_nat_gateway = true + enable_ipv6 = true + create_egress_only_igw = true + + public_subnet_ipv6_prefixes = [0, 1, 2] + public_subnet_assign_ipv6_address_on_creation = true + private_subnet_ipv6_prefixes = [3, 4, 5] + private_subnet_assign_ipv6_address_on_creation = true + intra_subnet_ipv6_prefixes = [6, 7, 8] + intra_subnet_assign_ipv6_address_on_creation = true public_subnet_tags = { "kubernetes.io/role/elb" = 1 diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 1b48556ce1..6ca254df27 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -35,7 +35,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| | [eks](#module\_eks) | ../.. | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | ## Resources diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index 1e928558c9..b67335c94e 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -6,7 +6,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.24" + cluster_version = "1.27" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -106,7 +106,7 @@ module "eks" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" + version = "~> 4.0" name = local.name cidr = local.vpc_cidr @@ -116,13 +116,8 @@ module "vpc" { public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - enable_flow_log = true - create_flow_log_cloudwatch_iam_role = true - create_flow_log_cloudwatch_log_group = true + enable_nat_gateway = true + single_nat_gateway = true public_subnet_tags = { "kubernetes.io/role/elb" = 1 diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 3335cfc4b3..d336b10b9d 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -73,7 +73,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|--------|---------| | [eks](#module\_eks) | ../.. | n/a | | [karpenter](#module\_karpenter) | ../../modules/karpenter | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | ## Resources diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index ac2ef4870e..a9c0e8832b 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -54,7 +54,7 @@ data "aws_ecrpublic_authorization_token" "token" { locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.24" + cluster_version = "1.27" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -286,7 +286,7 @@ resource "kubectl_manifest" "karpenter_example_deployment" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" + version = "~> 4.0" name = local.name cidr = local.vpc_cidr @@ -296,13 +296,8 @@ module "vpc" { public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - enable_flow_log = true - create_flow_log_cloudwatch_iam_role = true - create_flow_log_cloudwatch_log_group = true + enable_nat_gateway = true + single_nat_gateway = true public_subnet_tags = { "kubernetes.io/role/elb" = 1 diff --git a/examples/outposts/main.tf b/examples/outposts/main.tf index 5dc4797e0f..fc50b53c89 100644 --- a/examples/outposts/main.tf +++ b/examples/outposts/main.tf @@ -16,7 +16,7 @@ provider "kubernetes" { locals { name = "ex-${basename(path.cwd)}" - cluster_version = "1.21" # Required by EKS on Outposts + cluster_version = "1.27" # Required by EKS on Outposts outpost_arn = element(tolist(data.aws_outposts_outposts.this.arns), 0) instance_type = element(tolist(data.aws_outposts_outpost_instance_types.this.instance_types), 0) diff --git a/examples/outposts/prerequisites/main.tf b/examples/outposts/prerequisites/main.tf index e49d7bdbb5..014418121d 100644 --- a/examples/outposts/prerequisites/main.tf +++ b/examples/outposts/prerequisites/main.tf @@ -56,7 +56,7 @@ module "ssm_bastion_ec2" { rm terraform_${local.terraform_version}_linux_amd64.zip 2> /dev/null # Install kubectl - curl -LO https://dl.k8s.io/release/v1.21.0/bin/linux/amd64/kubectl + curl -LO https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubectl install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl # Remove default awscli which is v1 - we want latest v2 @@ -66,9 +66,8 @@ module "ssm_bastion_ec2" { ./aws/install # Clone repo - git clone https://github.com/bryantbiggs/terraform-aws-eks.git \ - && cd /home/ssm-user/terraform-aws-eks \ - && git checkout refactor/v19 + git clone https://github.com/terraform-aws-modules/terraform-aws-eks.git \ + && cd /home/ssm-user/terraform-aws-eks chown -R ssm-user:ssm-user /home/ssm-user/ EOT diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index d6944ac76d..c5ddbc325c 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -42,7 +42,7 @@ Note that this example may create resources which cost money. Run `terraform des | [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 1.5 | | [eks](#module\_eks) | ../.. | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | ## Resources diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 7f1993d05a..87be519086 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -19,7 +19,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.24" + cluster_version = "1.27" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -256,7 +256,7 @@ module "eks" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" + version = "~> 4.0" name = local.name cidr = local.vpc_cidr @@ -266,13 +266,8 @@ module "vpc" { public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - enable_flow_log = true - create_flow_log_cloudwatch_iam_role = true - create_flow_log_cloudwatch_log_group = true + enable_nat_gateway = true + single_nat_gateway = true public_subnet_tags = { "kubernetes.io/role/elb" = 1 diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 4bf3434663..bf3a35976a 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -10,7 +10,7 @@ module "eks_managed_node_group" { name = "separate-eks-mng" cluster_name = "my-cluster" - cluster_version = "1.24" + cluster_version = "1.27" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index e1c1abd92a..8e9b6dce99 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -168,7 +168,7 @@ No modules. | [irsa\_ssm\_parameter\_arns](#input\_irsa\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter | `list(string)` |
[
"arn:aws:ssm:*:*:parameter/aws/service/*"
]
| no | | [irsa\_subnet\_account\_id](#input\_irsa\_subnet\_account\_id) | Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account | `string` | `""` | no | | [irsa\_tag\_key](#input\_irsa\_tag\_key) | Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner | `string` | `"karpenter.sh/discovery"` | no | -| [irsa\_tag\_values](#input\_irsa\_tag\_values) | Tag values (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner. Defaults to cluster name when not set. | `list(string)` | `null` | no | +| [irsa\_tag\_values](#input\_irsa\_tag\_values) | Tag values (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner. Defaults to cluster name when not set. | `list(string)` | `[]` | no | | [irsa\_tags](#input\_irsa\_tags) | A map of additional tags to add the the IAM role for service accounts | `map(any)` | `{}` | no | | [irsa\_use\_name\_prefix](#input\_irsa\_use\_name\_prefix) | Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix | `bool` | `true` | no | | [policies](#input\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 2c809ed621..53ee84f818 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -64,7 +64,7 @@ resource "aws_iam_role" "irsa" { } locals { - irsa_tag_values = coalescelist([var.cluster_name], var.irsa_tag_values) + irsa_tag_values = coalescelist(var.irsa_tag_values, [var.cluster_name]) } data "aws_iam_policy_document" "irsa" { diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 47c42bace9..95a5a1df93 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -89,7 +89,7 @@ variable "irsa_tag_key" { variable "irsa_tag_values" { description = "Tag values (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner. Defaults to cluster name when not set." type = list(string) - default = null + default = [] } variable "irsa_ssm_parameter_arns" { diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index a9eff4772e..8964144994 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -10,7 +10,7 @@ module "self_managed_node_group" { name = "separate-self-mng" cluster_name = "my-cluster" - cluster_version = "1.24" + cluster_version = "1.27" cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" diff --git a/variables.tf b/variables.tf index b8577737d8..988b97970c 100644 --- a/variables.tf +++ b/variables.tf @@ -27,7 +27,7 @@ variable "cluster_name" { } variable "cluster_version" { - description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.24`)" + description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.27`)" type = string default = null } From b6fa04f65bdcb26869215fb840f5ee088a096bc8 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 30 May 2023 22:56:07 +0000 Subject: [PATCH 689/962] chore(release): version 19.15.2 [skip ci] ### [19.15.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.1...v19.15.2) (2023-05-30) ### Bug Fixes * Ensure `isra_tag_values` can be tried before defaulting to `cluster_name` on Karpenter module ([#2631](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2631)) ([6c56e2a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6c56e2ad20057a5672526b5484df96806598a4e2)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c933d4fe2c..1a9ef38e8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.15.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.1...v19.15.2) (2023-05-30) + + +### Bug Fixes + +* Ensure `isra_tag_values` can be tried before defaulting to `cluster_name` on Karpenter module ([#2631](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2631)) ([6c56e2a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6c56e2ad20057a5672526b5484df96806598a4e2)) + ### [19.15.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.0...v19.15.1) (2023-05-24) From a2fb28e63e923b1c46ae1f611c237cf1d47e96b4 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 7 Jun 2023 09:26:25 +0900 Subject: [PATCH 690/962] docs: Correct comment mis-spelling (#2642) --- examples/karpenter/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index a9c0e8832b..068ba6c2dd 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -93,13 +93,13 @@ module "eks" { resources = { limits = { cpu = "0.25" - # We are targetting the smallest Task size of 512Mb, so we subtract 256Mb from the + # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the # request/limit to ensure we can fit within that task memory = "256M" } requests = { cpu = "0.25" - # We are targetting the smallest Task size of 512Mb, so we subtract 256Mb from the + # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the # request/limit to ensure we can fit within that task memory = "256M" } From 6217d0eaab4c864ec4d40a31538e78a7fbcee5e3 Mon Sep 17 00:00:00 2001 From: sanjinp <34198911+sanjinp@users.noreply.github.com> Date: Fri, 9 Jun 2023 19:08:34 +0200 Subject: [PATCH 691/962] fix: Snapshot permissions issue for Karpenter submodule (#2649) fix: snapshot permirrions issue for karpenter submodule --- modules/karpenter/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 53ee84f818..19399ce99d 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -122,6 +122,7 @@ data "aws_iam_policy_document" "irsa" { actions = ["ec2:RunInstances"] resources = [ "arn:${local.partition}:ec2:*::image/*", + "arn:${local.partition}:ec2:*::snapshot/*", "arn:${local.partition}:ec2:*:${local.account_id}:instance/*", "arn:${local.partition}:ec2:*:${local.account_id}:spot-instances-request/*", "arn:${local.partition}:ec2:*:${local.account_id}:security-group/*", From 0a17f655fb7da00640627ed9255f1d96e42fcfd7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 9 Jun 2023 17:09:13 +0000 Subject: [PATCH 692/962] chore(release): version 19.15.3 [skip ci] ### [19.15.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.2...v19.15.3) (2023-06-09) ### Bug Fixes * Snapshot permissions issue for Karpenter submodule ([#2649](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2649)) ([6217d0e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6217d0eaab4c864ec4d40a31538e78a7fbcee5e3)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a9ef38e8a..062b48ec60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.15.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.2...v19.15.3) (2023-06-09) + + +### Bug Fixes + +* Snapshot permissions issue for Karpenter submodule ([#2649](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2649)) ([6217d0e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6217d0eaab4c864ec4d40a31538e78a7fbcee5e3)) + ### [19.15.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.1...v19.15.2) (2023-05-30) From c86f8d4db3236e7dae59ef9142da4d7e496138c8 Mon Sep 17 00:00:00 2001 From: Roberto Devesa <15369573+Roberdvs@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:58:01 +0200 Subject: [PATCH 693/962] fix: Use `coalesce` when desired default value is not `null` (#2696) Co-authored-by: Bryant Biggs --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 52d36aa201..ca0390ab7b 100644 --- a/main.tf +++ b/main.tf @@ -386,7 +386,7 @@ resource "aws_eks_addon" "this" { cluster_name = aws_eks_cluster.this[0].name addon_name = try(each.value.name, each.key) - addon_version = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) + addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) configuration_values = try(each.value.configuration_values, null) preserve = try(each.value.preserve, null) resolve_conflicts = try(each.value.resolve_conflicts, "OVERWRITE") @@ -414,7 +414,7 @@ resource "aws_eks_addon" "before_compute" { cluster_name = aws_eks_cluster.this[0].name addon_name = try(each.value.name, each.key) - addon_version = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) + addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) configuration_values = try(each.value.configuration_values, null) preserve = try(each.value.preserve, null) resolve_conflicts = try(each.value.resolve_conflicts, "OVERWRITE") From de14031c970fc23412ebbfae719cefe259c2ad32 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 27 Jul 2023 11:58:28 +0000 Subject: [PATCH 694/962] chore(release): version 19.15.4 [skip ci] ### [19.15.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.3...v19.15.4) (2023-07-27) ### Bug Fixes * Use `coalesce` when desired default value is not `null` ([#2696](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2696)) ([c86f8d4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c86f8d4db3236e7dae59ef9142da4d7e496138c8)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 062b48ec60..4d5a39d57d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.15.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.3...v19.15.4) (2023-07-27) + + +### Bug Fixes + +* Use `coalesce` when desired default value is not `null` ([#2696](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2696)) ([c86f8d4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c86f8d4db3236e7dae59ef9142da4d7e496138c8)) + ### [19.15.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.2...v19.15.3) (2023-06-09) From d4e6c153ad50359d896797a89f0f50fff96a636e Mon Sep 17 00:00:00 2001 From: Cody Grant Date: Thu, 3 Aug 2023 19:16:57 -0400 Subject: [PATCH 695/962] chore: Update FAQ for security group tagging (#2695) * Add aws_eks_addons.before_compute to the cluster_addons output * update FAQ docs for custom networking * Rephrasing --------- Co-authored-by: Bryant Biggs --- docs/faq.md | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 215cdb2086..3ca85e85e8 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -24,27 +24,7 @@ By default, EKS creates a cluster primary security group that is created outside attach_cluster_primary_security_group = true # default is false ``` -2. If you want to use the cluster primary security group, you can disable the tag passed to the node security group by overriding the tag expected value like: - -```hcl - attach_cluster_primary_security_group = true # default is false - - node_security_group_tags = { - "kubernetes.io/cluster/" = null # or any other value other than "owned" - } -``` - -3. By overriding the tag expected value on the cluster primary security group like: - -```hcl - attach_cluster_primary_security_group = true # default is false - - cluster_tags = { - "kubernetes.io/cluster/" = null # or any other value other than "owned" - } -``` - -4. By not attaching the cluster primary security group. The cluster primary security group has quite broad access and the module has instead provided a security group with the minimum amount of access to launch an empty EKS cluster successfully and users are encouraged to open up access when necessary to support their workload. +2. By not attaching the cluster primary security group. The cluster primary security group has quite broad access and the module has instead provided a security group with the minimum amount of access to launch an empty EKS cluster successfully and users are encouraged to open up access when necessary to support their workload. ```hcl attach_cluster_primary_security_group = false # this is the default for the module @@ -52,6 +32,8 @@ By default, EKS creates a cluster primary security group that is created outside In theory, if you are attaching the cluster primary security group, you shouldn't need to use the shared node security group created by the module. However, this is left up to users to decide for their requirements and use case. +If you choose to use [Custom Networking](https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html), make sure to only attach the security groups matching your choice above in your ENIConfig resources. This will ensure you avoid redundant tags. + ### Why are nodes not being registered? Nodes not being able to register with the EKS control plane is generally due to networking mis-configurations. From adb47f46dc53b1a0c18691a59dc58401c327c0be Mon Sep 17 00:00:00 2001 From: Angel Moctezuma <44302055+aamoctz@users.noreply.github.com> Date: Thu, 3 Aug 2023 16:23:36 -0700 Subject: [PATCH 696/962] feat: Add `node_iam_role_arns` local variable to check for Windows platform on EKS managed nodegroups (#2477) Update node_iam_role_arns locals to check for Windows Managed Node Groups Co-authored-by: Bryant Biggs --- main.tf | 3 ++- modules/eks-managed-node-group/README.md | 1 + modules/eks-managed-node-group/outputs.tf | 9 +++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index ca0390ab7b..1e73c760e9 100644 --- a/main.tf +++ b/main.tf @@ -469,7 +469,7 @@ locals { node_iam_role_arns_non_windows = distinct( compact( concat( - [for group in module.eks_managed_node_group : group.iam_role_arn], + [for group in module.eks_managed_node_group : group.iam_role_arn if group.platform != "windows"], [for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"], var.aws_auth_node_iam_role_arns_non_windows, ) @@ -479,6 +479,7 @@ locals { node_iam_role_arns_windows = distinct( compact( concat( + [for group in module.eks_managed_node_group : group.iam_role_arn if group.platform == "windows"], [for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"], var.aws_auth_node_iam_role_arns_windows, ) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index bf3a35976a..8dcb95f02b 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -193,4 +193,5 @@ module "eks_managed_node_group" { | [node\_group\_resources](#output\_node\_group\_resources) | List of objects containing information about underlying resources | | [node\_group\_status](#output\_node\_group\_status) | Status of the EKS Node Group | | [node\_group\_taints](#output\_node\_group\_taints) | List of objects containing information about taints applied to the node group | +| [platform](#output\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index 012cd46f4e..a72f27b347 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -88,3 +88,12 @@ output "iam_role_unique_id" { description = "Stable and unique string identifying the IAM role" value = try(aws_iam_role.this[0].unique_id, null) } + +################################################################################ +# Additional +################################################################################ + +output "platform" { + description = "Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based" + value = var.platform +} From 666603b6e531140d5d8fbd777cd90a7fbb8247dd Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 3 Aug 2023 23:24:04 +0000 Subject: [PATCH 697/962] chore(release): version 19.16.0 [skip ci] ## [19.16.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.4...v19.16.0) (2023-08-03) ### Features * Add `node_iam_role_arns` local variable to check for Windows platform on EKS managed nodegroups ([#2477](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2477)) ([adb47f4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/adb47f46dc53b1a0c18691a59dc58401c327c0be)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d5a39d57d..0d82002b29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.16.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.4...v19.16.0) (2023-08-03) + + +### Features + +* Add `node_iam_role_arns` local variable to check for Windows platform on EKS managed nodegroups ([#2477](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2477)) ([adb47f4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/adb47f46dc53b1a0c18691a59dc58401c327c0be)) + ### [19.15.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.3...v19.15.4) (2023-07-27) From 771465be280450fc96d889ef9e15f191bb512849 Mon Sep 17 00:00:00 2001 From: lfpalacios <30420888+lfpalacios@users.noreply.github.com> Date: Tue, 15 Aug 2023 17:26:25 -0300 Subject: [PATCH 698/962] docs: Correct `time_zone` argument on example autoscaling group schedule (#2722) --- examples/eks_managed_node_group/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 1fb1b8b723..b8097221bc 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -271,7 +271,7 @@ module "eks" { desired_size = 2 start_time = "2023-03-05T00:00:00Z" end_time = "2024-03-05T00:00:00Z" - timezone = "Etc/GMT+0" + time_zone = "Etc/GMT+0" recurrence = "0 0 * * *" }, scale-down = { @@ -280,7 +280,7 @@ module "eks" { desired_size = 0 start_time = "2023-03-05T12:00:00Z" end_time = "2024-03-05T12:00:00Z" - timezone = "Etc/GMT+0" + time_zone = "Etc/GMT+0" recurrence = "0 12 * * *" } } From 918aa7cc40cbc072836410747834de64d84f514d Mon Sep 17 00:00:00 2001 From: Brandon Wagner Date: Thu, 7 Sep 2023 07:11:34 -0500 Subject: [PATCH 699/962] docs: Correct Karpenter cfn link to IAM policy (#2736) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a87400e1c0..2d53af9a29 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Some of the addon/controller policies that are currently supported include: - [External DNS](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#iam-policy) - [External Secrets](https://github.com/external-secrets/kubernetes-external-secrets#add-a-secret) - [FSx for Lustre CSI Driver](https://github.com/kubernetes-sigs/aws-fsx-csi-driver/blob/master/docs/README.md) -- [Karpenter](https://github.com/aws/karpenter/blob/main/website/content/en/preview/getting-started/cloudformation.yaml) +- [Karpenter](https://github.com/aws/karpenter/blob/main/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml) - [Load Balancer Controller](https://github.com/kubernetes-sigs/aws-load-balancer-controller/blob/main/docs/install/iam_policy.json) - [Load Balancer Controller Target Group Binding Only](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/deploy/installation/#iam-permission-subset-for-those-who-use-targetgroupbinding-only-and-dont-plan-to-use-the-aws-load-balancer-controller-to-manage-security-group-rules) - [App Mesh Controller](https://github.com/aws/aws-app-mesh-controller-for-k8s/blob/master/config/iam/controller-iam-policy.json) From 5368a2bbd1bdb4e84e547ba111ef8477dfe36c08 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Fri, 6 Oct 2023 20:15:33 +0700 Subject: [PATCH 700/962] docs: Add IAM role state move to v18 upgrade doc (#2024) Co-authored-by: Bryant Biggs --- docs/UPGRADE-18.0.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/UPGRADE-18.0.md b/docs/UPGRADE-18.0.md index 3bfb672c3f..481e9fe021 100644 --- a/docs/UPGRADE-18.0.md +++ b/docs/UPGRADE-18.0.md @@ -13,6 +13,14 @@ cluster_security_group_name = $CLUSTER_NAME cluster_security_group_description = "EKS cluster security group." ``` +This configuration assumes that [`create_iam_role`](https://github.com/terraform-aws-modules/terraform-aws-eks#input_create_iam_role) is set to `true`, which is the default value. + +As the location of the Terraform state of the IAM role has been changed from 17.x to 18.x, you'll also have to move the state before running `terraform apply` by calling: + +``` +terraform state mv 'module.eks.aws_iam_role.cluster[0]' 'module.eks.aws_iam_role.this[0]' +``` + See more information [here](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1744#issuecomment-1027359982) ## List of backwards incompatible changes From feee18dd423b1e76f8a5119206f23306e5879b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Charest?= Date: Fri, 6 Oct 2023 09:20:13 -0400 Subject: [PATCH 701/962] feat: Add support for `allowed_instance_types` on self-managed nodegroup ASG (#2757) Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 2 +- README.md | 4 +- examples/complete/README.md | 4 +- examples/complete/versions.tf | 2 +- examples/eks_managed_node_group/README.md | 4 +- examples/eks_managed_node_group/versions.tf | 2 +- examples/fargate_profile/README.md | 4 +- examples/fargate_profile/versions.tf | 2 +- examples/karpenter/README.md | 6 +-- examples/karpenter/versions.tf | 2 +- examples/outposts/README.md | 4 +- examples/outposts/prerequisites/versions.tf | 2 +- examples/outposts/versions.tf | 2 +- examples/self_managed_node_group/README.md | 4 +- examples/self_managed_node_group/main.tf | 52 ++++++++++++++++++++ examples/self_managed_node_group/versions.tf | 2 +- modules/eks-managed-node-group/README.md | 4 +- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 +- modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 4 +- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/main.tf | 5 +- versions.tf | 2 +- 24 files changed, 88 insertions(+), 35 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dabb150840..b5adebd44a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.80.0 + rev: v1.83.4 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/README.md b/README.md index 2d53af9a29..88f8741b2c 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -232,7 +232,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | | [kubernetes](#provider\_kubernetes) | >= 2.10 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | diff --git a/examples/complete/README.md b/examples/complete/README.md index c03547b5e2..0fd43130da 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -34,14 +34,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | ## Modules diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index aeb892f359..d2ec4a8f9d 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 090273396c..c5278e07a0 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -30,14 +30,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | ## Modules diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index aeb892f359..d2ec4a8f9d 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 6ca254df27..9d41ed40b0 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -20,7 +20,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | | [helm](#requirement\_helm) | >= 2.7 | | [null](#requirement\_null) | >= 3.0 | @@ -28,7 +28,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | ## Modules diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index 17db7d8988..10d7878df7 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } helm = { source = "hashicorp/helm" diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index d336b10b9d..6549a3e8d2 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -52,7 +52,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 1.14 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | @@ -62,8 +62,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | -| [aws.virginia](#provider\_aws.virginia) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | +| [aws.virginia](#provider\_aws.virginia) | >= 4.57 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 1.14 | diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index cab7b21121..f1f0ee477e 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/outposts/README.md b/examples/outposts/README.md index 33684940ac..ff2542c4e9 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -42,14 +42,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | | [kubernetes](#provider\_kubernetes) | >= 2.10 | ## Modules diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf index 5f058b4c11..01d187af62 100644 --- a/examples/outposts/prerequisites/versions.tf +++ b/examples/outposts/prerequisites/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.34" + version = ">= 4.57" } } } diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf index aeb892f359..d2ec4a8f9d 100644 --- a/examples/outposts/versions.tf +++ b/examples/outposts/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index c5ddbc325c..5d849be0b9 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -26,14 +26,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | ## Modules diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 87be519086..d359226f3c 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -216,6 +216,58 @@ module "eks" { } } + instance_attributes = { + name = "instance-attributes" + + min_size = 1 + max_size = 2 + desired_size = 1 + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + instance_type = null + + # launch template configuration + instance_requirements = { + cpu_manufacturers = ["intel"] + instance_generations = ["current", "previous"] + spot_max_price_percentage_over_lowest_price = 100 + + vcpu_count = { + min = 1 + } + + allowed_instance_types = ["t*", "m*"] + } + + use_mixed_instances_policy = true + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 0 + on_demand_allocation_strategy = "lowest-price" + spot_allocation_strategy = "price-capacity-optimized" + } + + # ASG configuration + override = [ + { + instance_requirements = { + cpu_manufacturers = ["intel"] + instance_generations = ["current", "previous"] + spot_max_price_percentage_over_lowest_price = 100 + + vcpu_count = { + min = 1 + } + + allowed_instance_types = ["t*", "m*"] + } + } + ] + } + } + metadata_options = { http_endpoint = "enabled" http_tokens = "required" diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index aeb892f359..d2ec4a8f9d 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 8dcb95f02b..69d33ae51e 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 55eff62b09..01d187af62 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index cc0bab2a5b..d841524ac4 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 55eff62b09..01d187af62 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 8e9b6dce99..f7b57c0cb2 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -100,13 +100,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | +| [aws](#requirement\_aws) | >= 4.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.47 | +| [aws](#provider\_aws) | >= 4.57 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 55eff62b09..01d187af62 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } } } diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 049e32dfca..e29e52adb4 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -527,8 +527,9 @@ resource "aws_autoscaling_group" "this" { } } - accelerator_types = try(instance_requirements.value.accelerator_types, []) - bare_metal = try(instance_requirements.value.bare_metal, null) + accelerator_types = try(instance_requirements.value.accelerator_types, []) + allowed_instance_types = try(instance_requirements.value.allowed_instance_types, null) + bare_metal = try(instance_requirements.value.bare_metal, null) dynamic "baseline_ebs_bandwidth_mbps" { for_each = try([instance_requirements.value.baseline_ebs_bandwidth_mbps], []) diff --git a/versions.tf b/versions.tf index dfe867e876..e3bdcbf6d7 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.57" } tls = { source = "hashicorp/tls" From 386535f6c5c390c5cb3f46d982b1c6e57c6e22bc Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 6 Oct 2023 13:20:42 +0000 Subject: [PATCH 702/962] chore(release): version 19.17.0 [skip ci] ## [19.17.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.16.0...v19.17.0) (2023-10-06) ### Features * Add support for `allowed_instance_types` on self-managed nodegroup ASG ([#2757](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2757)) ([feee18d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/feee18dd423b1e76f8a5119206f23306e5879b26)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d82002b29..a542a7bacf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.17.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.16.0...v19.17.0) (2023-10-06) + + +### Features + +* Add support for `allowed_instance_types` on self-managed nodegroup ASG ([#2757](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2757)) ([feee18d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/feee18dd423b1e76f8a5119206f23306e5879b26)) + ## [19.16.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.15.4...v19.16.0) (2023-08-03) From 7e5de1566c7e1330c05c5e6c51f5ab4690001915 Mon Sep 17 00:00:00 2001 From: Daniel Frankcom Date: Fri, 6 Oct 2023 16:26:35 -0700 Subject: [PATCH 703/962] fix: Only include CA thumbprint in OIDC provider list (#2769) As per #2732 and #32847, only the CA thumbprint should be included in the OIDC provider thumbprint list. --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 1e73c760e9..674110e256 100644 --- a/main.tf +++ b/main.tf @@ -232,7 +232,7 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { count = local.create && var.enable_irsa && !local.create_outposts_local_cluster ? 1 : 0 client_id_list = distinct(compact(concat(["sts.${local.dns_suffix}"], var.openid_connect_audiences))) - thumbprint_list = concat(data.tls_certificate.this[0].certificates[*].sha1_fingerprint, var.custom_oidc_thumbprints) + thumbprint_list = concat([data.tls_certificate.this[0].certificates[0].sha1_fingerprint], var.custom_oidc_thumbprints) url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer tags = merge( From 4f635701ada43cd65642ba51cb7e5c689605a281 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 6 Oct 2023 23:27:09 +0000 Subject: [PATCH 704/962] chore(release): version 19.17.1 [skip ci] ### [19.17.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.0...v19.17.1) (2023-10-06) ### Bug Fixes * Only include CA thumbprint in OIDC provider list ([#2769](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2769)) ([7e5de15](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7e5de1566c7e1330c05c5e6c51f5ab4690001915)), closes [#2732](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2732) [#32847](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/32847) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a542a7bacf..a8a55d447f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.17.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.0...v19.17.1) (2023-10-06) + + +### Bug Fixes + +* Only include CA thumbprint in OIDC provider list ([#2769](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2769)) ([7e5de15](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7e5de1566c7e1330c05c5e6c51f5ab4690001915)), closes [#2732](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2732) [#32847](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/32847) + ## [19.17.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.16.0...v19.17.0) (2023-10-06) From f4766e5c27f060e8c7f5950cf82d1fe59c3231af Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 10 Oct 2023 11:57:29 -0400 Subject: [PATCH 705/962] fix: Karpenter node IAM role policies variable should be a map of strings, not list (#2771) --- .pre-commit-config.yaml | 2 +- examples/karpenter/main.tf | 10 ++++++++-- modules/karpenter/README.md | 7 ++++++- modules/karpenter/main.tf | 1 - modules/karpenter/variables.tf | 4 ++-- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b5adebd44a..15d3d3efd7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - '--args=--only=terraform_standard_module_structure' - '--args=--only=terraform_workspace_remote' - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 068ba6c2dd..b925de7855 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -160,7 +160,13 @@ module "karpenter" { cluster_name = module.eks.cluster_name irsa_oidc_provider_arn = module.eks.oidc_provider_arn - policies = { + # Used to attach additional IAM policies to the Karpenter controller IRSA role + # policies = { + # "xxx" = "yyy" + # } + + # Used to attach additional IAM policies to the Karpenter node IAM role + iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } @@ -176,7 +182,7 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "v0.21.1" + version = "v0.29.0" set { name = "settings.aws.clusterName" diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index f7b57c0cb2..4a7f9c7f74 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -42,6 +42,11 @@ module "karpenter" { irsa_oidc_provider_arn = module.eks.oidc_provider_arn irsa_namespace_service_accounts = ["karpenter:karpenter"] + # Attach additional IAM policies to the Karpenter node IAM role + iam_role_additional_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + tags = { Environment = "dev" Terraform = "true" @@ -146,7 +151,7 @@ No modules. | [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `true` | no | | [create\_irsa](#input\_create\_irsa) | Determines whether an IAM role for service accounts is created | `bool` | `true` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | -| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 19399ce99d..223c400b5d 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -224,7 +224,6 @@ data "aws_iam_policy_document" "queue" { "sqs.${local.dns_suffix}", ] } - } } diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 95a5a1df93..c025237b36 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -223,8 +223,8 @@ variable "iam_role_attach_cni_policy" { variable "iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" - type = list(string) - default = [] + type = map(string) + default = {} } variable "iam_role_tags" { From 51cc6bec880ac8dc361b60a4b05d5f2bcd98eb6a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 10 Oct 2023 15:58:10 +0000 Subject: [PATCH 706/962] chore(release): version 19.17.2 [skip ci] ### [19.17.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.1...v19.17.2) (2023-10-10) ### Bug Fixes * Karpenter node IAM role policies variable should be a map of strings, not list ([#2771](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2771)) ([f4766e5](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f4766e5c27f060e8c7f5950cf82d1fe59c3231af)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8a55d447f..ece010ff38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.17.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.1...v19.17.2) (2023-10-10) + + +### Bug Fixes + +* Karpenter node IAM role policies variable should be a map of strings, not list ([#2771](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2771)) ([f4766e5](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f4766e5c27f060e8c7f5950cf82d1fe59c3231af)) + ### [19.17.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.0...v19.17.1) (2023-10-06) From bd4bda266e23635c7ca09b6e9d307b29ef6b8579 Mon Sep 17 00:00:00 2001 From: Lama <9804647+Lamaspanzer@users.noreply.github.com> Date: Mon, 30 Oct 2023 18:41:24 +0100 Subject: [PATCH 707/962] fix: Correct key used on `license_configuration_arn` (#2796) fix: Reference to undeclared resource Co-authored-by: Lamaspanzer --- modules/eks-managed-node-group/main.tf | 4 ++-- modules/self-managed-node-group/main.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 1d3fe81698..14c9b48a98 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -176,10 +176,10 @@ resource "aws_launch_template" "this" { key_name = var.key_name dynamic "license_specification" { - for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} + for_each = length(var.license_specifications) > 0 ? var.license_specifications : [] content { - license_configuration_arn = license_specifications.value.license_configuration_arn + license_configuration_arn = license_specification.value.license_configuration_arn } } diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index e29e52adb4..5dc1bcca1e 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -282,10 +282,10 @@ resource "aws_launch_template" "this" { key_name = var.key_name dynamic "license_specification" { - for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} + for_each = length(var.license_specifications) > 0 ? var.license_specifications : [] content { - license_configuration_arn = license_specifications.value.license_configuration_arn + license_configuration_arn = license_specification.value.license_configuration_arn } } From f0aa0dac8670e85ce5443994c4b0af2c0afd27fb Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 30 Oct 2023 17:41:53 +0000 Subject: [PATCH 708/962] chore(release): version 19.17.3 [skip ci] ### [19.17.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.2...v19.17.3) (2023-10-30) ### Bug Fixes * Correct key used on `license_configuration_arn` ([#2796](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2796)) ([bd4bda2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/bd4bda266e23635c7ca09b6e9d307b29ef6b8579)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ece010ff38..a4c166172b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.17.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.2...v19.17.3) (2023-10-30) + + +### Bug Fixes + +* Correct key used on `license_configuration_arn` ([#2796](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2796)) ([bd4bda2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/bd4bda266e23635c7ca09b6e9d307b29ef6b8579)) + ### [19.17.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.1...v19.17.2) (2023-10-10) From ba0ebeb11a64a6400a3666165509975d5cdfea43 Mon Sep 17 00:00:00 2001 From: JulianTheKalamari Date: Mon, 30 Oct 2023 14:40:03 -0400 Subject: [PATCH 709/962] fix: Updating license_specification result type (#2798) --- modules/eks-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 14c9b48a98..e4a59758b4 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -176,7 +176,7 @@ resource "aws_launch_template" "this" { key_name = var.key_name dynamic "license_specification" { - for_each = length(var.license_specifications) > 0 ? var.license_specifications : [] + for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} content { license_configuration_arn = license_specification.value.license_configuration_arn diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 5dc1bcca1e..3b04db8b80 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -282,7 +282,7 @@ resource "aws_launch_template" "this" { key_name = var.key_name dynamic "license_specification" { - for_each = length(var.license_specifications) > 0 ? var.license_specifications : [] + for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} content { license_configuration_arn = license_specification.value.license_configuration_arn From bb8aae31ec555f586a3ff482e35f10deb9b1a30b Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 30 Oct 2023 18:40:44 +0000 Subject: [PATCH 710/962] chore(release): version 19.17.4 [skip ci] ### [19.17.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.3...v19.17.4) (2023-10-30) ### Bug Fixes * Updating license_specification result type ([#2798](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2798)) ([ba0ebeb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ba0ebeb11a64a6400a3666165509975d5cdfea43)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c166172b..2c9c576a15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.17.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.3...v19.17.4) (2023-10-30) + + +### Bug Fixes + +* Updating license_specification result type ([#2798](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2798)) ([ba0ebeb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ba0ebeb11a64a6400a3666165509975d5cdfea43)) + ### [19.17.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.2...v19.17.3) (2023-10-30) From 69eb4569b971c236682a79be9dd7f0db326bac1e Mon Sep 17 00:00:00 2001 From: knkcni <33808993+knkcni@users.noreply.github.com> Date: Tue, 31 Oct 2023 21:01:34 +0100 Subject: [PATCH 711/962] docs: Add ebs disk in complete example (#2773) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 📝 (complete) Add ebs disk in complete example Co-authored-by: Bryant Biggs --- examples/complete/main.tf | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index e946906821..184dba2228 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -189,6 +189,19 @@ module "eks" { } } + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = 100 + volume_type = "gp3" + iops = 3000 + throughput = 150 + delete_on_termination = true + } + } + } + update_config = { max_unavailable_percentage = 33 # or set `max_unavailable` } From aec2bab1d8da89b65b84d11fef77cbc969fccc91 Mon Sep 17 00:00:00 2001 From: Michael Barrientos Date: Wed, 1 Nov 2023 08:33:07 -0700 Subject: [PATCH 712/962] feat: Add Karpenter v1beta1 compatibility (#2800) * feat: Add Karpenter v1beta1 compatibility * fix: Update to make changes opt-in * fix: Update resource schemas to align with latest Karpenter version - validated and working as intended --------- Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 2 +- examples/karpenter/README.md | 6 +- examples/karpenter/main.tf | 110 +++++++++++++++++---------------- modules/karpenter/README.md | 1 + modules/karpenter/main.tf | 20 +++++- modules/karpenter/variables.tf | 6 ++ 6 files changed, 86 insertions(+), 59 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 15d3d3efd7..145baf94e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.4 + rev: v1.83.5 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 6549a3e8d2..06b29e7bb4 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -73,7 +73,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|--------|---------| | [eks](#module\_eks) | ../.. | n/a | | [karpenter](#module\_karpenter) | ../../modules/karpenter | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | ## Resources @@ -81,8 +81,8 @@ Note that this example may create resources which cost money. Run `terraform des |------|------| | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_node_template](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_node_class](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index b925de7855..b8d7be97ff 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -160,10 +160,9 @@ module "karpenter" { cluster_name = module.eks.cluster_name irsa_oidc_provider_arn = module.eks.oidc_provider_arn - # Used to attach additional IAM policies to the Karpenter controller IRSA role - # policies = { - # "xxx" = "yyy" - # } + # In v0.32.0/v1beta1, Karpenter now creates the IAM instance profile + # so we disable the Terraform creation and add the necessary permissions for Karpenter IRSA + enable_karpenter_instance_profile_creation = true # Used to attach additional IAM policies to the Karpenter node IAM role iam_role_additional_policies = { @@ -182,51 +181,38 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "v0.29.0" - - set { - name = "settings.aws.clusterName" - value = module.eks.cluster_name - } - - set { - name = "settings.aws.clusterEndpoint" - value = module.eks.cluster_endpoint - } - - set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.karpenter.irsa_arn - } - - set { - name = "settings.aws.defaultInstanceProfile" - value = module.karpenter.instance_profile_name - } - - set { - name = "settings.aws.interruptionQueueName" - value = module.karpenter.queue_name - } + version = "v0.32.1" + + values = [ + <<-EOT + settings: + clusterName: ${module.eks.cluster_name} + clusterEndpoint: ${module.eks.cluster_endpoint} + interruptionQueueName: ${module.karpenter.queue_name} + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: ${module.karpenter.irsa_arn} + EOT + ] } -resource "kubectl_manifest" "karpenter_provisioner" { +resource "kubectl_manifest" "karpenter_node_class" { yaml_body = <<-YAML - apiVersion: karpenter.sh/v1alpha5 - kind: Provisioner + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass metadata: name: default spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] - limits: - resources: - cpu: 1000 - providerRef: - name: default - ttlSecondsAfterEmpty: 30 + amiFamily: AL2 + role: ${module.karpenter.role_name} + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: ${module.eks.cluster_name} + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: ${module.eks.cluster_name} + tags: + karpenter.sh/discovery: ${module.eks.cluster_name} YAML depends_on = [ @@ -234,23 +220,39 @@ resource "kubectl_manifest" "karpenter_provisioner" { ] } -resource "kubectl_manifest" "karpenter_node_template" { +resource "kubectl_manifest" "karpenter_node_pool" { yaml_body = <<-YAML - apiVersion: karpenter.k8s.aws/v1alpha1 - kind: AWSNodeTemplate + apiVersion: karpenter.sh/v1beta1 + kind: NodePool metadata: name: default spec: - subnetSelector: - karpenter.sh/discovery: ${module.eks.cluster_name} - securityGroupSelector: - karpenter.sh/discovery: ${module.eks.cluster_name} - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} + template: + spec: + nodeClassRef: + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + limits: + cpu: 1000 + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 30s YAML depends_on = [ - helm_release.karpenter + kubectl_manifest.karpenter_node_class ] } @@ -292,7 +294,7 @@ resource "kubectl_manifest" "karpenter_example_deployment" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 4.0" + version = "~> 5.0" name = local.name cidr = local.vpc_cidr diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 4a7f9c7f74..7b813471ad 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -150,6 +150,7 @@ No modules. | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `true` | no | | [create\_irsa](#input\_create\_irsa) | Determines whether an IAM role for service accounts is created | `bool` | `true` | no | +| [enable\_karpenter\_instance\_profile\_creation](#input\_enable\_karpenter\_instance\_profile\_creation) | Determines whether Karpenter will be allowed to create the IAM instance profile (v1beta1) or if Terraform will (v1alpha1) | `bool` | `false` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 223c400b5d..5cf4d39fd3 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -160,6 +160,24 @@ data "aws_iam_policy_document" "irsa" { resources = [aws_sqs_queue.this[0].arn] } } + + # TODO - this will be replaced in v20.0 with the scoped policy provided by Karpenter + # https://github.com/aws/karpenter/blob/main/website/content/en/docs/upgrading/v1beta1-controller-policy.json + dynamic "statement" { + for_each = var.enable_karpenter_instance_profile_creation ? [1] : [] + + content { + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:GetInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:TagInstanceProfile", + ] + resources = ["*"] + } + } } resource "aws_iam_policy" "irsa" { @@ -368,7 +386,7 @@ locals { } resource "aws_iam_instance_profile" "this" { - count = var.create && var.create_instance_profile ? 1 : 0 + count = var.create && var.create_instance_profile && !var.enable_karpenter_instance_profile_creation ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index c025237b36..4a8389671b 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -123,6 +123,12 @@ variable "irsa_assume_role_condition_test" { default = "StringEquals" } +variable "enable_karpenter_instance_profile_creation" { + description = "Determines whether Karpenter will be allowed to create the IAM instance profile (v1beta1) or if Terraform will (v1alpha1)" + type = bool + default = false +} + ################################################################################ # Node Termination Queue ################################################################################ From e4c50987f714b0924cfc07ff2855a8edc3eddea1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 1 Nov 2023 15:33:35 +0000 Subject: [PATCH 713/962] chore(release): version 19.18.0 [skip ci] ## [19.18.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.4...v19.18.0) (2023-11-01) ### Features * Add Karpenter v1beta1 compatibility ([#2800](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2800)) ([aec2bab](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/aec2bab1d8da89b65b84d11fef77cbc969fccc91)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c9c576a15..5784d93752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.18.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.4...v19.18.0) (2023-11-01) + + +### Features + +* Add Karpenter v1beta1 compatibility ([#2800](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2800)) ([aec2bab](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/aec2bab1d8da89b65b84d11fef77cbc969fccc91)) + ### [19.17.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.3...v19.17.4) (2023-10-30) From 0732bea85f46fd2629705f9ee5f87cb695ee95e5 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 4 Nov 2023 08:41:59 -0400 Subject: [PATCH 714/962] feat: Update KMS module to avoid calling data sources when `create_kms_key = false` (#2804) --- README.md | 2 +- main.tf | 2 +- modules/karpenter/main.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 88f8741b2c..5c53966497 100644 --- a/README.md +++ b/README.md @@ -243,7 +243,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|--------|---------| | [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ./modules/eks-managed-node-group | n/a | | [fargate\_profile](#module\_fargate\_profile) | ./modules/fargate-profile | n/a | -| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 1.1.0 | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 2.1.0 | | [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ./modules/self-managed-node-group | n/a | ## Resources diff --git a/main.tf b/main.tf index 674110e256..e96ce90876 100644 --- a/main.tf +++ b/main.tf @@ -122,7 +122,7 @@ resource "aws_cloudwatch_log_group" "this" { module "kms" { source = "terraform-aws-modules/kms/aws" - version = "1.1.0" # Note - be mindful of Terraform/provider version compatibility between modules + version = "2.1.0" # Note - be mindful of Terraform/provider version compatibility between modules create = local.create && var.create_kms_key && local.enable_cluster_encryption_config # not valid on Outposts diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 5cf4d39fd3..f74d577643 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -324,7 +324,7 @@ locals { iam_role_name = coalesce(var.iam_role_name, "Karpenter-${var.cluster_name}") iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" - cni_policy = var.cluster_ip_family == "ipv6" ? "${local.iam_role_policy_prefix}/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${local.partition}:iam::${local.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } data "aws_iam_policy_document" "assume_role" { From dfed830957079301b879814e87608728576dd168 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 4 Nov 2023 12:42:29 +0000 Subject: [PATCH 715/962] chore(release): version 19.19.0 [skip ci] ## [19.19.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.18.0...v19.19.0) (2023-11-04) ### Features * Update KMS module to avoid calling data sources when `create_kms_key = false` ([#2804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2804)) ([0732bea](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0732bea85f46fd2629705f9ee5f87cb695ee95e5)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5784d93752..4bbb619610 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.19.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.18.0...v19.19.0) (2023-11-04) + + +### Features + +* Update KMS module to avoid calling data sources when `create_kms_key = false` ([#2804](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2804)) ([0732bea](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0732bea85f46fd2629705f9ee5f87cb695ee95e5)) + ## [19.18.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.17.4...v19.18.0) (2023-11-01) From c36c8dc825aa09e2ded20ff675905aa8857853cf Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 10 Nov 2023 08:59:00 -0500 Subject: [PATCH 716/962] fix: Remove additional conditional on Karpenter instance profile creation to support upgrading (#2812) --- modules/karpenter/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index f74d577643..2ce9f95c16 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -386,7 +386,7 @@ locals { } resource "aws_iam_instance_profile" "this" { - count = var.create && var.create_instance_profile && !var.enable_karpenter_instance_profile_creation ? 1 : 0 + count = var.create && var.create_instance_profile ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null From f0e2e8badf0bb61a450f6c95a3a53211912b3004 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 10 Nov 2023 13:59:23 +0000 Subject: [PATCH 717/962] chore(release): version 19.19.1 [skip ci] ### [19.19.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.19.0...v19.19.1) (2023-11-10) ### Bug Fixes * Remove additional conditional on Karpenter instance profile creation to support upgrading ([#2812](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2812)) ([c36c8dc](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c36c8dc825aa09e2ded20ff675905aa8857853cf)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bbb619610..0456955d8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [19.19.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.19.0...v19.19.1) (2023-11-10) + + +### Bug Fixes + +* Remove additional conditional on Karpenter instance profile creation to support upgrading ([#2812](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2812)) ([c36c8dc](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c36c8dc825aa09e2ded20ff675905aa8857853cf)) + ## [19.19.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.18.0...v19.19.0) (2023-11-04) From 091c68051d9cbf24644121a24c715307f00c44b3 Mon Sep 17 00:00:00 2001 From: Daniel Frankcom Date: Tue, 14 Nov 2023 07:07:47 -0800 Subject: [PATCH 718/962] feat: Allow OIDC root CA thumbprint to be included/excluded (#2778) * fix: Allow OIDC root CA thumbprint to be included/excluded * chore: Consolidate conditional logic within local conditional --------- Co-authored-by: Bryant Biggs --- README.md | 1 + main.tf | 13 ++++++++++--- variables.tf | 6 ++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5c53966497..465bc79878 100644 --- a/README.md +++ b/README.md @@ -345,6 +345,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [include\_oidc\_root\_ca\_thumbprint](#input\_include\_oidc\_root\_ca\_thumbprint) | Determines whether to include the root CA thumbprint in the OpenID Connect (OIDC) identity provider's server certificate(s) | `bool` | `true` | no | | [kms\_key\_administrators](#input\_kms\_key\_administrators) | A list of IAM ARNs for [key administrators](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-administrators). If no value is provided, the current caller identity is used to ensure at least one key admin is available | `list(string)` | `[]` | no | | [kms\_key\_aliases](#input\_kms\_key\_aliases) | A list of aliases to create. Note - due to the use of `toset()`, values must be static strings and not computed values | `list(string)` | `[]` | no | | [kms\_key\_deletion\_window\_in\_days](#input\_kms\_key\_deletion\_window\_in\_days) | The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key. If you specify a value, it must be between `7` and `30`, inclusive. If you do not specify a value, it defaults to `30` | `number` | `null` | no | diff --git a/main.tf b/main.tf index e96ce90876..60e48ab6c4 100644 --- a/main.tf +++ b/main.tf @@ -220,19 +220,26 @@ resource "aws_security_group_rule" "cluster" { # Note - this is different from EKS identity provider ################################################################################ +locals { + # Not available on outposts + create_oidc_provider = local.create && var.enable_irsa && !local.create_outposts_local_cluster + + oidc_root_ca_thumbprint = local.create_oidc_provider && var.include_oidc_root_ca_thumbprint ? [data.tls_certificate.this[0].certificates[0].sha1_fingerprint] : [] +} + data "tls_certificate" "this" { # Not available on outposts - count = local.create && var.enable_irsa && !local.create_outposts_local_cluster ? 1 : 0 + count = local.create_oidc_provider && var.include_oidc_root_ca_thumbprint ? 1 : 0 url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer } resource "aws_iam_openid_connect_provider" "oidc_provider" { # Not available on outposts - count = local.create && var.enable_irsa && !local.create_outposts_local_cluster ? 1 : 0 + count = local.create_oidc_provider ? 1 : 0 client_id_list = distinct(compact(concat(["sts.${local.dns_suffix}"], var.openid_connect_audiences))) - thumbprint_list = concat([data.tls_certificate.this[0].certificates[0].sha1_fingerprint], var.custom_oidc_thumbprints) + thumbprint_list = concat(local.oidc_root_ca_thumbprint, var.custom_oidc_thumbprints) url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer tags = merge( diff --git a/variables.tf b/variables.tf index 988b97970c..17488000c1 100644 --- a/variables.tf +++ b/variables.tf @@ -358,6 +358,12 @@ variable "openid_connect_audiences" { default = [] } +variable "include_oidc_root_ca_thumbprint" { + description = "Determines whether to include the root CA thumbprint in the OpenID Connect (OIDC) identity provider's server certificate(s)" + type = bool + default = true +} + variable "custom_oidc_thumbprints" { description = "Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s)" type = list(string) From ec454c51cb0026254b68cafe7aa9d05f873af990 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 14 Nov 2023 15:08:15 +0000 Subject: [PATCH 719/962] chore(release): version 19.20.0 [skip ci] ## [19.20.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.19.1...v19.20.0) (2023-11-14) ### Features * Allow OIDC root CA thumbprint to be included/excluded ([#2778](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2778)) ([091c680](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/091c68051d9cbf24644121a24c715307f00c44b3)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0456955d8c..3cd9c324c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.20.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.19.1...v19.20.0) (2023-11-14) + + +### Features + +* Allow OIDC root CA thumbprint to be included/excluded ([#2778](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2778)) ([091c680](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/091c68051d9cbf24644121a24c715307f00c44b3)) + ### [19.19.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.19.0...v19.19.1) (2023-11-10) From 4c5c97b5d404a4e46945e3b6228d469743669937 Mon Sep 17 00:00:00 2001 From: BlacksmithJoe <93652830+BlacksmithJoe@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:54:19 +0100 Subject: [PATCH 720/962] feat: Add tags for CloudWatch log group only (#2841) * feature: added cloudwatch log group tags input * fix: terraform docs run --- README.md | 1 + main.tf | 1 + variables.tf | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 465bc79878..bd15fcbae5 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws\_auth\_users](#input\_aws\_auth\_users) | List of user maps to add to the aws-auth configmap | `list(any)` | `[]` | no | | [cloudwatch\_log\_group\_kms\_key\_id](#input\_cloudwatch\_log\_group\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `null` | no | | [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days | `number` | `90` | no | +| [cloudwatch\_log\_group\_tags](#input\_cloudwatch\_log\_group\_tags) | A map of additional tags to add to the cloudwatch log group created | `map(string)` | `{}` | no | | [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | | [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no | | [cluster\_addons\_timeouts](#input\_cluster\_addons\_timeouts) | Create, update, and delete timeout configurations for the cluster addons | `map(string)` | `{}` | no | diff --git a/main.tf b/main.tf index 60e48ab6c4..6a25f3a63f 100644 --- a/main.tf +++ b/main.tf @@ -112,6 +112,7 @@ resource "aws_cloudwatch_log_group" "this" { tags = merge( var.tags, + var.cloudwatch_log_group_tags, { Name = "/aws/eks/${var.cluster_name}/cluster" } ) } diff --git a/variables.tf b/variables.tf index 17488000c1..810cee2bda 100644 --- a/variables.tf +++ b/variables.tf @@ -228,6 +228,12 @@ variable "cloudwatch_log_group_kms_key_id" { default = null } +variable "cloudwatch_log_group_tags" { + description = "A map of additional tags to add to the cloudwatch log group created" + type = map(string) + default = {} +} + ################################################################################ # Cluster Security Group ################################################################################ From 2cb1fac31b0fc2dd6a236b0c0678df75819c5a3b Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 11 Dec 2023 14:54:50 +0000 Subject: [PATCH 721/962] chore(release): version 19.21.0 [skip ci] ## [19.21.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.20.0...v19.21.0) (2023-12-11) ### Features * Add tags for CloudWatch log group only ([#2841](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2841)) ([4c5c97b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/4c5c97b5d404a4e46945e3b6228d469743669937)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cd9c324c3..bcfa09ae70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [19.21.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.20.0...v19.21.0) (2023-12-11) + + +### Features + +* Add tags for CloudWatch log group only ([#2841](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2841)) ([4c5c97b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/4c5c97b5d404a4e46945e3b6228d469743669937)) + ## [19.20.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.19.1...v19.20.0) (2023-11-14) From 6b40bdbb1d283d9259f43b03d24dca99cc1eceff Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 2 Feb 2024 09:36:25 -0500 Subject: [PATCH 722/962] feat!: Replace the use of `aws-auth` configmap with EKS cluster access entry (#2858) * feat: Replace `resolve_conflicts` with `resolve_conflicts_on_create`/`delete`; raise MSV of AWS provider to `v5.0` to support * fix: Replace dynamic DNS suffix for `sts:AssumeRole` API calls for static suffix * feat: Add module tag * feat: Align Karpenter permissions with Karpenter v1beta1/v0.32 permissions from upstream * refactor: Move `aws-auth` ConfigMap functionality to its own sub-module * chore: Update examples * feat: Add state `moved` block for Karpenter Pod Identity role re-name * fix: Correct variable `create` description * feat: Add support for cluster access entries * chore: Bump MSV of Terraform to `1.3` * fix: Replace defunct kubectl provider with an updated forked equivalent * chore: Update and validate examples for access entry; clean up provider usage * docs: Correct double redundant variable descriptions * feat: Add support for Cloudwatch log group class argument * fix: Update usage tag placement, fix Karpenter event spelling, add upcoming changes section to upgrade guide * feat: Update Karpenter module to generalize naming used and align policy with the upstream Karpenter policy * feat: Add native support for Windows based managed nodegroups similar to AL2 and Bottlerocket * feat: Update self-managed nodegroup module to use latest features of ASG * docs: Update and simplify docs * fix: Correct variable description for AMI types * fix: Update upgrade guide with changes; rename Karpenter controller resource names to support migrating for users * docs: Complete upgrade guide docs for migration and changes applied * Update examples/karpenter/README.md Co-authored-by: Anton Babenko * Update examples/outposts/README.md Co-authored-by: Anton Babenko * Update modules/karpenter/README.md Co-authored-by: Anton Babenko --------- Co-authored-by: Anton Babenko --- .pre-commit-config.yaml | 3 +- README.md | 165 +----- docs/README.md | 2 +- docs/UPGRADE-18.0.md | 2 +- docs/UPGRADE-19.0.md | 2 +- docs/UPGRADE-20.0.md | 262 +++++++++ docs/faq.md | 8 - docs/irsa_integration.md | 84 --- docs/user_data.md | 1 + examples/complete/README.md | 107 ---- examples/complete/main.tf | 482 ----------------- examples/complete/outputs.tf | 192 ------- examples/complete/versions.tf | 14 - examples/eks_managed_node_group/README.md | 18 +- examples/eks_managed_node_group/main.tf | 173 ++---- examples/eks_managed_node_group/outputs.tf | 18 +- examples/eks_managed_node_group/versions.tf | 8 +- examples/fargate_profile/README.md | 14 +- examples/fargate_profile/main.tf | 91 ++-- examples/fargate_profile/outputs.tf | 18 +- examples/fargate_profile/versions.tf | 12 +- examples/karpenter/README.md | 38 +- examples/karpenter/main.tf | 71 ++- examples/karpenter/outputs.tf | 50 +- examples/karpenter/versions.tf | 16 +- examples/outposts/README.md | 19 +- examples/outposts/main.tf | 21 +- examples/outposts/outputs.tf | 18 +- examples/outposts/prerequisites/main.tf | 6 +- examples/outposts/prerequisites/versions.tf | 4 +- examples/outposts/versions.tf | 6 +- examples/self_managed_node_group/README.md | 15 +- examples/self_managed_node_group/main.tf | 81 +-- examples/self_managed_node_group/outputs.tf | 18 +- examples/self_managed_node_group/versions.tf | 8 +- examples/user_data/README.md | 10 +- examples/user_data/main.tf | 67 ++- examples/user_data/outputs.tf | 21 + examples/user_data/versions.tf | 2 +- main.tf | 254 +++++---- modules/_user_data/README.md | 2 +- modules/_user_data/main.tf | 4 +- modules/_user_data/versions.tf | 2 +- modules/aws-auth/README.md | 81 +++ modules/aws-auth/main.tf | 47 ++ .../aws-auth/outputs.tf | 0 modules/aws-auth/variables.tf | 39 ++ modules/aws-auth/versions.tf | 10 + modules/eks-managed-node-group/README.md | 8 +- modules/eks-managed-node-group/main.tf | 2 +- modules/eks-managed-node-group/variables.tf | 2 +- modules/eks-managed-node-group/versions.tf | 4 +- modules/fargate-profile/README.md | 6 +- modules/fargate-profile/versions.tf | 4 +- modules/karpenter/README.md | 145 +++-- modules/karpenter/main.tf | 510 +++++++++++++----- modules/karpenter/migrations.tf | 56 ++ modules/karpenter/outputs.tf | 47 +- modules/karpenter/variables.tf | 147 ++--- modules/karpenter/versions.tf | 4 +- modules/self-managed-node-group/README.md | 11 +- modules/self-managed-node-group/main.tf | 82 ++- modules/self-managed-node-group/outputs.tf | 27 +- modules/self-managed-node-group/variables.tf | 54 +- modules/self-managed-node-group/versions.tf | 4 +- node_groups.tf | 26 +- outputs.tf | 25 +- templates/aws_auth_cm.tpl | 37 -- templates/windows_user_data.tpl | 4 + variables.tf | 96 ++-- versions.tf | 8 +- 71 files changed, 1784 insertions(+), 2111 deletions(-) create mode 100644 docs/UPGRADE-20.0.md delete mode 100644 docs/irsa_integration.md delete mode 100644 examples/complete/README.md delete mode 100644 examples/complete/main.tf delete mode 100644 examples/complete/outputs.tf delete mode 100644 examples/complete/versions.tf create mode 100644 modules/aws-auth/README.md create mode 100644 modules/aws-auth/main.tf rename examples/complete/variables.tf => modules/aws-auth/outputs.tf (100%) create mode 100644 modules/aws-auth/variables.tf create mode 100644 modules/aws-auth/versions.tf create mode 100644 modules/karpenter/migrations.tf delete mode 100644 templates/aws_auth_cm.tpl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 145baf94e4..93900a8e02 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.5 + rev: v1.86.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -27,3 +27,4 @@ repos: hooks: - id: check-merge-conflict - id: end-of-file-fixer + - id: trailing-whitespace diff --git a/README.md b/README.md index bd15fcbae5..1039fa6c82 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ Terraform module which creates AWS EKS (Kubernetes) resources - [Frequently Asked Questions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md) - [Compute Resources](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/compute_resources.md) -- [IRSA Integration](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/irsa_integration.md) - [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/user_data.md) - [Network Connectivity](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/network_connectivity.md) - Upgrade Guides @@ -27,58 +26,17 @@ Please note that we strive to provide a comprehensive suite of documentation for The examples provided under `examples/` provide a comprehensive suite of configurations that demonstrate nearly all of the possible different configurations and settings that can be used with this module. However, these examples are not representative of clusters that you would normally find in use for production workloads. For reference architectures that utilize this module, please see the following: - [EKS Reference Architecture](https://github.com/clowdhaus/eks-reference-architecture) - -## Available Features - -- AWS EKS Cluster Addons -- AWS EKS Identity Provider Configuration -- [AWS EKS on Outposts support](https://aws.amazon.com/blogs/aws/deploy-your-amazon-eks-clusters-locally-on-aws-outposts/) -- All [node types](https://docs.aws.amazon.com/eks/latest/userguide/eks-compute.html) are supported: - - [EKS Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) - - [Self Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) - - [Fargate Profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) -- Support for creating Karpenter related AWS infrastructure resources (e.g. IAM roles, SQS queue, EventBridge rules, etc.) -- Support for custom AMI, custom launch template, and custom user data including custom user data template -- Support for Amazon Linux 2 EKS Optimized AMI and Bottlerocket nodes - - Windows based node support is limited to a default user data template that is provided due to the lack of Windows support and manual steps required to provision Windows based EKS nodes -- Support for module created security group, bring your own security groups, as well as adding additional security group rules to the module created security group(s) -- Support for creating node groups/profiles separate from the cluster through the use of sub-modules (same as what is used by root module) -- Support for node group/profile "default" settings - useful for when creating multiple node groups/Fargate profiles where you want to set a common set of configurations once, and then individually control only select features on certain node groups/profiles - -### [IRSA Terraform Module](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) - -An IAM role for service accounts (IRSA) sub-module has been created to make deploying common addons/controllers easier. Instead of users having to create a custom IAM role with the necessary federated role assumption required for IRSA plus find and craft the associated policy required for the addon/controller, users can create the IRSA role and policy with a few lines of code. See the [`terraform-aws-iam/examples/iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/examples/iam-role-for-service-accounts-eks/main.tf) directory for examples on how to use the IRSA sub-module in conjunction with this (`terraform-aws-eks`) module. - -Some of the addon/controller policies that are currently supported include: - -- [Cert-Manager](https://cert-manager.io/docs/configuration/acme/dns01/route53/#set-up-an-iam-role) -- [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md) -- [EBS CSI Driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/example-iam-policy.json) -- [EFS CSI Driver](https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/iam-policy-example.json) -- [External DNS](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#iam-policy) -- [External Secrets](https://github.com/external-secrets/kubernetes-external-secrets#add-a-secret) -- [FSx for Lustre CSI Driver](https://github.com/kubernetes-sigs/aws-fsx-csi-driver/blob/master/docs/README.md) -- [Karpenter](https://github.com/aws/karpenter/blob/main/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml) -- [Load Balancer Controller](https://github.com/kubernetes-sigs/aws-load-balancer-controller/blob/main/docs/install/iam_policy.json) - - [Load Balancer Controller Target Group Binding Only](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/deploy/installation/#iam-permission-subset-for-those-who-use-targetgroupbinding-only-and-dont-plan-to-use-the-aws-load-balancer-controller-to-manage-security-group-rules) -- [App Mesh Controller](https://github.com/aws/aws-app-mesh-controller-for-k8s/blob/master/config/iam/controller-iam-policy.json) - - [App Mesh Envoy Proxy](https://raw.githubusercontent.com/aws/aws-app-mesh-controller-for-k8s/master/config/iam/envoy-iam-policy.json) -- [Managed Service for Prometheus](https://docs.aws.amazon.com/prometheus/latest/userguide/set-up-irsa.html) -- [Node Termination Handler](https://github.com/aws/aws-node-termination-handler#5-create-an-iam-role-for-the-pods) -- [Velero](https://github.com/vmware-tanzu/velero-plugin-for-aws#option-1-set-permissions-with-an-iam-user) -- [VPC CNI](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html) - -See [terraform-aws-iam/modules/iam-role-for-service-accounts](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) for current list of supported addon/controller policies as more are added to the project. +- [EKS Blueprints](https://github.com/aws-ia/terraform-aws-eks-blueprints) ## Usage ```hcl module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 19.0" + version = "~> 20.0" cluster_name = "my-cluster" - cluster_version = "1.27" + cluster_version = "1.29" cluster_endpoint_public_access = true @@ -98,51 +56,13 @@ module "eks" { subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] control_plane_subnet_ids = ["subnet-xyzde987", "subnet-slkjf456", "subnet-qeiru789"] - # Self Managed Node Group(s) - self_managed_node_group_defaults = { - instance_type = "m6i.large" - update_launch_template_default_version = true - iam_role_additional_policies = { - AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" - } - } - - self_managed_node_groups = { - one = { - name = "mixed-1" - max_size = 5 - desired_size = 2 - - use_mixed_instances_policy = true - mixed_instances_policy = { - instances_distribution = { - on_demand_base_capacity = 0 - on_demand_percentage_above_base_capacity = 10 - spot_allocation_strategy = "capacity-optimized" - } - - override = [ - { - instance_type = "m5.large" - weighted_capacity = "1" - }, - { - instance_type = "m6i.large" - weighted_capacity = "2" - }, - ] - } - } - } - # EKS Managed Node Group(s) eks_managed_node_group_defaults = { instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] } eks_managed_node_groups = { - blue = {} - green = { + example = { min_size = 1 max_size = 10 desired_size = 1 @@ -152,47 +72,6 @@ module "eks" { } } - # Fargate Profile(s) - fargate_profiles = { - default = { - name = "default" - selectors = [ - { - namespace = "default" - } - ] - } - } - - # aws-auth configmap - manage_aws_auth_configmap = true - - aws_auth_roles = [ - { - rolearn = "arn:aws:iam::66666666666:role/role1" - username = "role1" - groups = ["system:masters"] - }, - ] - - aws_auth_users = [ - { - userarn = "arn:aws:iam::66666666666:user/user1" - username = "user1" - groups = ["system:masters"] - }, - { - userarn = "arn:aws:iam::66666666666:user/user2" - username = "user2" - groups = ["system:masters"] - }, - ] - - aws_auth_accounts = [ - "777777777777", - "888888888888", - ] - tags = { Environment = "dev" Terraform = "true" @@ -202,7 +81,6 @@ module "eks" { ## Examples -- [Complete](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete): EKS Cluster using all available node group types in various combinations demonstrating many of the supported features and configurations - [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group): EKS Cluster using EKS managed node groups - [Fargate Profile](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile): EKS cluster using [Fargate Profiles](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) - [Karpenter](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/karpenter): EKS Cluster with [Karpenter](https://karpenter.sh/) provisioned for intelligent data plane management @@ -222,9 +100,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.34 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -232,8 +109,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.57 | -| [kubernetes](#provider\_kubernetes) | >= 2.10 | +| [aws](#provider\_aws) | >= 5.34 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | @@ -252,6 +128,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|------| | [aws_cloudwatch_log_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | | [aws_ec2_tag.cluster_primary_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ec2_tag) | resource | +| [aws_eks_access_entry.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | +| [aws_eks_access_policy_association.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_policy_association) | resource | | [aws_eks_addon.before_compute](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_addon.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | @@ -267,8 +145,6 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_security_group.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group_rule.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | -| [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource | | [time_sleep.this](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | @@ -282,13 +158,10 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [access\_entries](#input\_access\_entries) | Map of access entries to add to the cluster | `any` | `{}` | no | | [attach\_cluster\_encryption\_policy](#input\_attach\_cluster\_encryption\_policy) | Indicates whether or not to attach an additional policy for the cluster IAM role to utilize the encryption key provided | `bool` | `true` | no | -| [aws\_auth\_accounts](#input\_aws\_auth\_accounts) | List of account maps to add to the aws-auth configmap | `list(any)` | `[]` | no | -| [aws\_auth\_fargate\_profile\_pod\_execution\_role\_arns](#input\_aws\_auth\_fargate\_profile\_pod\_execution\_role\_arns) | List of Fargate profile pod execution role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no | -| [aws\_auth\_node\_iam\_role\_arns\_non\_windows](#input\_aws\_auth\_node\_iam\_role\_arns\_non\_windows) | List of non-Windows based node IAM role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no | -| [aws\_auth\_node\_iam\_role\_arns\_windows](#input\_aws\_auth\_node\_iam\_role\_arns\_windows) | List of Windows based node IAM role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no | -| [aws\_auth\_roles](#input\_aws\_auth\_roles) | List of role maps to add to the aws-auth configmap | `list(any)` | `[]` | no | -| [aws\_auth\_users](#input\_aws\_auth\_users) | List of user maps to add to the aws-auth configmap | `list(any)` | `[]` | no | +| [authentication\_mode](#input\_authentication\_mode) | The authentication mode for the cluster. Valid values are `CONFIG_MAP`, `API` or `API_AND_CONFIG_MAP` | `string` | `"API_AND_CONFIG_MAP"` | no | +| [cloudwatch\_log\_group\_class](#input\_cloudwatch\_log\_group\_class) | Specified the log class of the log group. Possible values are: `STANDARD` or `INFREQUENT_ACCESS` | `string` | `null` | no | | [cloudwatch\_log\_group\_kms\_key\_id](#input\_cloudwatch\_log\_group\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `null` | no | | [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days | `number` | `90` | no | | [cloudwatch\_log\_group\_tags](#input\_cloudwatch\_log\_group\_tags) | A map of additional tags to add to the cloudwatch log group created | `map(string)` | `{}` | no | @@ -305,7 +178,6 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `true` | no | | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `false` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | -| [cluster\_iam\_role\_dns\_suffix](#input\_cluster\_iam\_role\_dns\_suffix) | Base DNS domain name for the current partition (e.g., amazonaws.com in AWS Commercial, amazonaws.com.cn in AWS China) | `string` | `null` | no | | [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | @@ -321,8 +193,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.27`) | `string` | `null` | no | | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | -| [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | -| [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | +| [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | | [create\_cluster\_primary\_security\_group\_tags](#input\_create\_cluster\_primary\_security\_group\_tags) | Indicates whether or not to tag the cluster's primary security group. This security group is created by the EKS service, not the module, and therefore tagging is handled after cluster creation | `bool` | `true` | no | | [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default | `bool` | `true` | no | @@ -334,8 +205,9 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed nodegroup(s), self-managed nodegroup(s), Fargate profile(s)) | `string` | `"30s"` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | +| [enable\_cluster\_creator\_admin\_permissions](#input\_enable\_cluster\_creator\_admin\_permissions) | Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | -| [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled. Defaults to `true` | `bool` | `true` | no | +| [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled | `bool` | `true` | no | | [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no | | [fargate\_profiles](#input\_fargate\_profiles) | Map of Fargate Profile definitions to create | `any` | `{}` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | @@ -351,13 +223,12 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [kms\_key\_aliases](#input\_kms\_key\_aliases) | A list of aliases to create. Note - due to the use of `toset()`, values must be static strings and not computed values | `list(string)` | `[]` | no | | [kms\_key\_deletion\_window\_in\_days](#input\_kms\_key\_deletion\_window\_in\_days) | The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key. If you specify a value, it must be between `7` and `30`, inclusive. If you do not specify a value, it defaults to `30` | `number` | `null` | no | | [kms\_key\_description](#input\_kms\_key\_description) | The description of the key as viewed in AWS console | `string` | `null` | no | -| [kms\_key\_enable\_default\_policy](#input\_kms\_key\_enable\_default\_policy) | Specifies whether to enable the default key policy. Defaults to `false` | `bool` | `false` | no | +| [kms\_key\_enable\_default\_policy](#input\_kms\_key\_enable\_default\_policy) | Specifies whether to enable the default key policy | `bool` | `true` | no | | [kms\_key\_override\_policy\_documents](#input\_kms\_key\_override\_policy\_documents) | List of IAM policy documents that are merged together into the exported document. In merging, statements with non-blank `sid`s will override statements with the same `sid` | `list(string)` | `[]` | no | | [kms\_key\_owners](#input\_kms\_key\_owners) | A list of IAM ARNs for those who will have full key permissions (`kms:*`) | `list(string)` | `[]` | no | | [kms\_key\_service\_users](#input\_kms\_key\_service\_users) | A list of IAM ARNs for [key service users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-service-integration) | `list(string)` | `[]` | no | | [kms\_key\_source\_policy\_documents](#input\_kms\_key\_source\_policy\_documents) | List of IAM policy documents that are merged together into the exported document. Statements must have unique `sid`s | `list(string)` | `[]` | no | | [kms\_key\_users](#input\_kms\_key\_users) | A list of IAM ARNs for [key users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-users) | `list(string)` | `[]` | no | -| [manage\_aws\_auth\_configmap](#input\_manage\_aws\_auth\_configmap) | Determines whether to manage the aws-auth configmap | `bool` | `false` | no | | [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no | | [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | | [node\_security\_group\_enable\_recommended\_rules](#input\_node\_security\_group\_enable\_recommended\_rules) | Determines whether to enable recommended security group rules for the node security group created. This includes node-to-node TCP ingress on ephemeral ports and allows all egress traffic | `bool` | `true` | no | @@ -379,7 +250,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Description | |------|-------------| -| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | [DEPRECATED - use `var.manage_aws_auth_configmap`] Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | | [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | | [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | @@ -416,7 +287,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple ## License -Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-rds-aurora/tree/master/LICENSE) for full details. +Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. ## Additional information for users from Russia and Belarus diff --git a/docs/README.md b/docs/README.md index 889b8481e4..144826b4cf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,10 +4,10 @@ - [Frequently Asked Questions](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md) - [Compute Resources](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/compute_resources.md) -- [IRSA Integration](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/irsa_integration.md) - [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/user_data.md) - [Network Connectivity](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/network_connectivity.md) - Upgrade Guides - [Upgrade to v17.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-17.0.md) - [Upgrade to v18.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-18.0.md) - [Upgrade to v19.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-19.0.md) + - [Upgrade to v20.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-20.0.md) diff --git a/docs/UPGRADE-18.0.md b/docs/UPGRADE-18.0.md index 481e9fe021..3f25ca8e43 100644 --- a/docs/UPGRADE-18.0.md +++ b/docs/UPGRADE-18.0.md @@ -13,7 +13,7 @@ cluster_security_group_name = $CLUSTER_NAME cluster_security_group_description = "EKS cluster security group." ``` -This configuration assumes that [`create_iam_role`](https://github.com/terraform-aws-modules/terraform-aws-eks#input_create_iam_role) is set to `true`, which is the default value. +This configuration assumes that [`create_iam_role`](https://github.com/terraform-aws-modules/terraform-aws-eks#input_create_iam_role) is set to `true`, which is the default value. As the location of the Terraform state of the IAM role has been changed from 17.x to 18.x, you'll also have to move the state before running `terraform apply` by calling: diff --git a/docs/UPGRADE-19.0.md b/docs/UPGRADE-19.0.md index 19bd5f3946..5bb74723f4 100644 --- a/docs/UPGRADE-19.0.md +++ b/docs/UPGRADE-19.0.md @@ -58,7 +58,7 @@ Please consult the `examples` directory for reference example configurations. If ### Variable and output changes 1. Removed variables: - + - `node_security_group_ntp_ipv4_cidr_block` - default security group settings have an egress rule for ALL to `0.0.0.0/0`/`::/0` - `node_security_group_ntp_ipv6_cidr_block` - default security group settings have an egress rule for ALL to `0.0.0.0/0`/`::/0` - Self-managed node groups: diff --git a/docs/UPGRADE-20.0.md b/docs/UPGRADE-20.0.md new file mode 100644 index 0000000000..72bdd57699 --- /dev/null +++ b/docs/UPGRADE-20.0.md @@ -0,0 +1,262 @@ +# Upgrade from v19.x to v20.x + +Please consult the `examples` directory for reference example configurations. If you find a bug, please open an issue with supporting configuration to reproduce. + +## List of backwards incompatible changes + +- Minium supported AWS provider version increased to `v5.34` +- Minimum supported Terraform version increased to `v1.3` to support Terraform state `moved` blocks as well as other advanced features +- The `resolve_conflicts` argument within the `cluster_addons` configuration has been replaced with `resolve_conflicts_on_create` and `resolve_conflicts_on_delete` now that `resolve_conflicts` is deprecated +- The default/fallback value for the `preserve` argument of `cluster_addons`is now set to `true`. This has shown to be useful for users deprovisioning clusters while avoiding the situation where the CNI is deleted too early and causes resources to be left orphaned resulting in conflicts. +- The Karpenter sub-module's use of the `irsa` naming convention has been removed, along with an update to the Karpenter controller IAM policy to align with Karpenter's `v1beta1`/`v0.32` changes. Instead of referring to the role as `irsa` or `pod_identity`, its simply just an IAM role used by the Karpenter controller and there is support for use with either IRSA and/or Pod Identity (default) at this time +- The `aws-auth` ConfigMap resources have been moved to a standalone sub-module. This removes the Kubernetes provider requirement from the main module and allows for the `aws-auth` ConfigMap to be managed independently of the main module. This sub-module will be removed entirely in the next major release. +- Support for cluster access management has been added with the default authentication mode set as `API_AND_CONFIG_MAP`. This is a one way change if applied; if you wish to use `CONFIG_MAP`, you will need to set `authentication_mode = "CONFIG_MAP"` explicitly when upgrading. +- Karpenter EventBridge rule key `spot_interrupt` updated to correct mis-spelling (was `spot_interupt`). This will cause the rule to be replaced + +### ⚠️ Upcoming Changes Planned in v21.0 ⚠️ + +To give users advanced notice and provide some future direction for this module, these are the following changes we will be looking to make in the next major release of this module: + +1. The `aws-auth` sub-module will be removed entirely from the project. Since this sub-module is captured in the v20.x releases, users can continue using it even after the module moves forward with the next major version. The long term strategy and direction is cluster access entry and to rely only on the AWS Terraform provider. +2. The default value for `authentication_mode` will change to `API`. Aligning with point 1 above, this is a one way change, but users are free to specify the value of their choosing in place of this default (when the change is made). This module will proceed with an EKS API first strategy. +3. The launch template and autoscaling group usage contained within the EKS managed nodegroup and self-managed nodegroup sub-modules *might be replaced with the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module. At minimum, it makes sense to replace most of functionality in the self-managed nodegroup module with this external module, but its not yet clear if there is any benefit of using it in the EKS managed nodegroup sub-module. The interface that users interact with will stay the same, the changes will be internal to the implementation and we will do everything we can to keep the disruption to a minimum. + +## Additional changes + +### Added + + - A module tag has been added to the cluster control plane + - Support for cluster access entries. The `bootstrap_cluster_creator_admin_permissions` setting on the control plane has been hardcoded to `false` since this operation is a one time operation only at cluster creation per the EKS API. Instead, users can enable/disable `enable_cluster_creator_admin_permissions` at any time to achieve the same functionality. This takes the identity that Terraform is using to make API calls and maps it into a cluster admin via an access entry. For users on existing clusters, you will need to remove the default cluster administrator that was created by EKS prior to the cluster access entry APIs - see the section [`Removing the default cluster administrator`](https://aws.amazon.com/blogs/containers/a-deep-dive-into-simplified-amazon-eks-access-management-controls/) for more details. + - Support for specifying the CloudWatch log group class (standard or infrequent access) + - Native support for Windows based managed nodegroups similar to AL2 and Bottlerocket + - Self-managed nodegroups now support `instance_maintenance_policy` and have added `max_healthy_percentage`, `scale_in_protected_instances`, and `standby_instances` arguments to the `instance_refresh.preferences` block + +### Modified + + - For `sts:AssumeRole` permissions by services, the use of dynamically looking up the DNS suffix has been replaced with the static value of `amazonaws.com`. This does not appear to change by partition and instead requires users to set this manually for non-commercial regions. + - The default value for `kms_key_enable_default_policy` has changed from `false` to `true` to align with the default behavior of the `aws_kms_key` resource + - The Karpenter default value for `create_instance_profile` has changed from `true` to `false` to align with the changes in Karpenter v0.32 + - The Karpenter variable `create_instance_profile` default value has changed from `true` to `false`. Starting with Karpenter `v0.32.0`, Karpenter accepts an IAM role and creates the EC2 instance profile used by the nodes + +### Removed + + - The `complete` example has been removed due to its redundancy with the other examples + - References to the IRSA sub-module in the IAM repository have been removed. Once https://github.com/clowdhaus/terraform-aws-eks-pod-identity has been updated and moved into the organization, the documentation here will be updated to mention the new module. + +### Variable and output changes + +1. Removed variables: + + - `cluster_iam_role_dns_suffix` - replaced with a static string of `amazonaws.com` + - `manage_aws_auth_configmap` + - `create_aws_auth_configmap` + - `aws_auth_node_iam_role_arns_non_windows` + - `aws_auth_node_iam_role_arns_windows` + - `aws_auth_fargate_profile_pod_execution_role_arn` + - `aws_auth_roles` + - `aws_auth_users` + - `aws_auth_accounts` + + - Karpenter + - `irsa_tag_key` + - `irsa_tag_values` + - `irsa_subnet_account_id` + - `enable_karpenter_instance_profile_creation` + +2. Renamed variables: + + - Karpenter + - `create_irsa` -> `create_iam_role` + - `irsa_name` -> `iam_role_name` + - `irsa_use_name_prefix` -> `iam_role_name_prefix` + - `irsa_path` -> `iam_role_path` + - `irsa_description` -> `iam_role_description` + - `irsa_max_session_duration` -> `iam_role_max_session_duration` + - `irsa_permissions_boundary_arn` -> `iam_role_permissions_boundary_arn` + - `irsa_tags` -> `iam_role_tags` + - `policies` -> `iam_role_policies` + - `irsa_policy_name` -> `iam_policy_name` + - `irsa_ssm_parameter_arns` -> `ami_id_ssm_parameter_arns` + - `create_iam_role` -> `create_node_iam_role` + - `iam_role_additional_policies` -> `node_iam_role_additional_policies` + - `policies` -> `iam_role_policies` + - `iam_role_arn` -> `node_iam_role_arn` + - `iam_role_name` -> `node_iam_role_name` + - `iam_role_name_prefix` -> `node_iam_role_name_prefix` + - `iam_role_path` -> `node_iam_role_path` + - `iam_role_description` -> `node_iam_role_description` + - `iam_role_max_session_duration` -> `node_iam_role_max_session_duration` + - `iam_role_permissions_boundary_arn` -> `node_iam_role_permissions_boundary_arn` + - `iam_role_attach_cni_policy` -> `node_iam_role_attach_cni_policy` + - `iam_role_additional_policies` -> `node_iam_role_additional_policies` + - `iam_role_tags` -> `node_iam_role_tags` + +3. Added variables: + + - `create_access_entry` + - `enable_cluster_creator_admin_permissions` + - `authentication_mode` + - `access_entries` + - `cloudwatch_log_group_class` + + - Karpenter + - `iam_policy_name` + - `iam_policy_use_name_prefix` + - `iam_policy_description` + - `iam_policy_path` + - `enable_irsa` + - `create_access_entry` + - `access_entry_type` + + - Self-managed nodegroup + - `instance_maintenance_policy` + - `create_access_entry` + - `iam_role_arn` + +4. Removed outputs: + + - `aws_auth_configmap_yaml` + +5. Renamed outputs: + + - Karpenter + - `irsa_name` -> `iam_role_name` + - `irsa_arn` -> `iam_role_arn` + - `irsa_unique_id` -> `iam_role_unique_id` + - `role_name` -> `node_iam_role_name` + - `role_arn` -> `node_iam_role_arn` + - `role_unique_id` -> `node_iam_role_unique_id` + +6. Added outputs: + + - `access_entries` + + - Karpenter + - `node_access_entry_arn` + + - Self-managed nodegroup + - `access_entry_arn` + +## Upgrade Migrations + +### Diff of Before (v19.21) vs After (v20.0) + +```diff + module "eks" { + source = "terraform-aws-modules/eks/aws" +- version = "~> 19.21" ++ version = "~> 20.0" + +# If you want to maintain the current default behavior of v19.x ++ kms_key_enable_default_policy = false + +- manage_aws_auth_configmap = true + +- aws_auth_roles = [ +- { +- rolearn = "arn:aws:iam::66666666666:role/role1" +- username = "role1" +- groups = ["custom-role-group"] +- }, +- ] + +- aws_auth_users = [ +- { +- userarn = "arn:aws:iam::66666666666:user/user1" +- username = "user1" +- groups = ["custom-users-group"] +- }, +- ] +} + ++ module "eks" { ++ source = "terraform-aws-modules/eks/aws//modules/aws-auth" ++ version = "~> 20.0" + ++ manage_aws_auth_configmap = true + ++ aws_auth_roles = [ ++ { ++ rolearn = "arn:aws:iam::66666666666:role/role1" ++ username = "role1" ++ groups = ["custom-role-group"] ++ }, ++ ] + ++ aws_auth_users = [ ++ { ++ userarn = "arn:aws:iam::66666666666:user/user1" ++ username = "user1" ++ groups = ["custom-users-group"] ++ }, ++ ] ++ } +``` + +### Karpenter Diff of Before (v19.21) vs After (v20.0) + +```diff + module "eks" { + source = "terraform-aws-modules/eks/aws//modules/karpenter" +- version = "~> 19.21" ++ version = "~> 20.0" + +# If you wish to maintain the current default behavior of v19.x ++ enable_irsa = true ++ create_instance_profile = true + +# To avoid any resource re-creation ++ iam_role_name = "KarpenterIRSA-${module.eks.cluster_name}" ++ iam_role_description = "Karpenter IAM role for service account" ++ iam_policy_name = "KarpenterIRSA-${module.eks.cluster_name}" ++ iam_policy_description = "Karpenter IAM role for service account" +} +``` + +## Terraform State Moves + +#### ⚠️ Authentication Mode Changes ⚠️ + +Changing the `authentication_mode` is a one-way decision. See [announcement blog](https://aws.amazon.com/blogs/containers/a-deep-dive-into-simplified-amazon-eks-access-management-controls/) for further details: + +> Switching authentication modes on an existing cluster is a one-way operation. You can switch from CONFIG_MAP to API_AND_CONFIG_MAP. You can then switch from API_AND_CONFIG_MAP to API. You cannot revert these operations in the opposite direction. Meaning you cannot switch back to CONFIG_MAP or API_AND_CONFIG_MAP from API. And you cannot switch back to CONFIG_MAP from API_AND_CONFIG_MAP. + +### authentication_mode = "CONFIG_MAP" + +If using `authentication_mode = "CONFIG_MAP"`, before making any changes, you will first need to remove the configmap from the statefile to avoid any disruptions: + +```sh +terraform state rm 'module.eks.kubernetes_config_map_v1_data.aws_auth[0]' +terraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]' # include if Terraform created the original configmap +``` + +Once the configmap has been removed from the statefile, you can add the new `aws-auth` sub-module and copy the relevant definitions from the EKS module over to the new `aws-auth` sub-module definition (see before after diff above). + +#### ⚠️ Node IAM Roles + +You will need to add entries for any IAM roles used by nodegroups and/or Fargate profiles - the module no longer handles this in the background on behalf of users. + +When you apply the changes with the new sub-module, the configmap in the cluster will get updated with the contents provided in the sub-module definition, so please be sure all of the necessary entries are added before applying the changes. + +### authentication_mode = "API_AND_CONFIG_MAP" + +When using `authentication_mode = "API_AND_CONFIG_MAP"` and there are entries that will remain in the configmap (entries that cannot be replaced by cluster access entry), you will first need to update the `authentication_mode` on the cluster to `"API_AND_CONFIG_MAP"`. To help make this upgrade process easier, a copy of the changes defined in the [`v20.0.0`](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2858) PR have been captured [here](https://github.com/clowdhaus/terraform-aws-eks-v20-migrate) but with the `aws-auth` components still provided in the module. This means you get the equivalent of the `v20.0.0` module, but it still includes support for the `aws-auth` configmap. You can follow the provided README on that interim migration module for the order of execution and return here once the `authentication_mode` has been updated to `"API_AND_CONFIG_MAP"`. Note - EKS automatically adds access entries for the roles used by EKS managed nodegroups and Fargate profiles; users do not need to do anything additional for these roles. + +Once the `authentication_mode` has been updated, next you will need to remove the configmap from the statefile to avoid any disruptions: + +```sh +terraform state rm 'module.eks.kubernetes_config_map_v1_data.aws_auth[0]' +terraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]' # include if Terraform created the original configmap +``` + +#### ℹ️ Terraform 1.7+ users + +If you are using Terraform `v1.7+`, you can utilize the [`remove`](https://developer.hashicorp.com/terraform/language/resources/syntax#removing-resources) to facilitate both the removal of the configmap through code. You can create a fork/clone of the provided [migration module](https://github.com/clowdhaus/terraform-aws-eks-migrate-v19-to-v20) and add the `remove` blocks and apply those changes before proceeding. We do not want to force users onto the bleeding edge with this module, so we have not included `remove` support at this time. + +Once the configmap has been removed from the statefile, you can add the new `aws-auth` sub-module and copy the relevant definitions from the EKS module over to the new `aws-auth` sub-module definition (see before after diff above). When you apply the changes with the new sub-module, the configmap in the cluster will get updated with the contents provided in the sub-module definition, so please be sure all of the necessary entries are added before applying the changes. In the before/example above - the configmap would remove any entries for roles used by nodegroups and/or Fargate Profiles, but maintain the custom entries for users and roles passed into the module definition. + +### authentication_mode = "API" + +In order to switch to `API` only using cluster access entry, you first need to update the `authentication_mode` on the cluster to `API_AND_CONFIG_MAP` without modifying the `aws-auth` configmap. To help make this upgrade process easier, a copy of the changes defined in the [`v20.0.0`](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2858) PR have been captured [here](https://github.com/clowdhaus/terraform-aws-eks-v20-migrate) but with the `aws-auth` components still provided in the module. This means you get the equivalent of the `v20.0.0` module, but it still includes support for the `aws-auth` configmap. You can follow the provided README on that interim migration module for the order of execution and return here once the `authentication_mode` has been updated to `"API_AND_CONFIG_MAP"`. Note - EKS automatically adds access entries for the roles used by EKS managed nodegroups and Fargate profiles; users do not need to do anything additional for these roles. + +Once the `authentication_mode` has been updated, you can update the `authentication_mode` on the cluster to `API` and remove the `aws-auth` configmap components. diff --git a/docs/faq.md b/docs/faq.md index 3ca85e85e8..395c8c3130 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -57,14 +57,6 @@ If you require a public endpoint, setting up both (public and private) and restr The module is configured to ignore this value. Unfortunately, Terraform does not support variables within the `lifecycle` block. The setting is ignored to allow autoscaling via controllers such as cluster autoscaler or Karpenter to work properly and without interference by Terraform. Changing the desired count must be handled outside of Terraform once the node group is created. -### How can I deploy Windows based nodes? - -To enable Windows support for your EKS cluster, you will need to apply some configuration manually. See the [Enabling Windows Support (Windows/MacOS/Linux)](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support). - -In addition, Windows based nodes require an additional cluster RBAC role (`eks:kube-proxy-windows`). - -Note: Windows based node support is limited to a default user data template that is provided due to the lack of Windows support and manual steps required to provision Windows based EKS nodes. - ### How do I access compute resource attributes? Examples of accessing the attributes of the compute resource(s) created by the root module are shown below. Note - the assumption is that your cluster module definition is named `eks` as in `module "eks" { ... }`: diff --git a/docs/irsa_integration.md b/docs/irsa_integration.md deleted file mode 100644 index cc6a549500..0000000000 --- a/docs/irsa_integration.md +++ /dev/null @@ -1,84 +0,0 @@ - -### IRSA Integration - -An [IAM role for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) module has been created to work in conjunction with this module. The [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/tree/master/modules/iam-role-for-service-accounts-eks) module has a set of pre-defined IAM policies for common addons. Check [`policy.tf`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/modules/iam-role-for-service-accounts-eks/policies.tf) for a list of the policies currently supported. One example of this integration is shown below, and more can be found in the [`iam-role-for-service-accounts`](https://github.com/terraform-aws-modules/terraform-aws-iam/blob/master/examples/iam-role-for-service-accounts-eks/main.tf) example directory: - -```hcl -module "eks" { - source = "terraform-aws-modules/eks/aws" - - cluster_name = "example" - cluster_version = "1.27" - - cluster_addons = { - vpc-cni = { - resolve_conflicts = "OVERWRITE" - service_account_role_arn = module.vpc_cni_irsa.iam_role_arn - } - } - - vpc_id = "vpc-1234556abcdef" - subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] - - eks_managed_node_group_defaults = { - # We are using the IRSA created below for permissions - # However, we have to provision a new cluster with the policy attached FIRST - # before we can disable. Without this initial policy, - # the VPC CNI fails to assign IPs and nodes cannot join the new cluster - iam_role_attach_cni_policy = true - } - - eks_managed_node_groups = { - default = {} - } - - tags = { - Environment = "dev" - Terraform = "true" - } -} - -module "vpc_cni_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - - role_name = "vpc_cni" - attach_vpc_cni_policy = true - vpc_cni_enable_ipv4 = true - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:aws-node"] - } - } - - tags = { - Environment = "dev" - Terraform = "true" - } -} - -module "karpenter_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - - role_name = "karpenter_controller" - attach_karpenter_controller_policy = true - - karpenter_controller_cluster_id = module.eks.cluster_id - karpenter_controller_node_iam_role_arns = [ - module.eks.eks_managed_node_groups["default"].iam_role_arn - ] - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["karpenter:karpenter"] - } - } - - tags = { - Environment = "dev" - Terraform = "true" - } -} -``` diff --git a/docs/user_data.md b/docs/user_data.md index 7b236944ff..87a28a2b47 100644 --- a/docs/user_data.md +++ b/docs/user_data.md @@ -8,6 +8,7 @@ Users can see the various methods of using and providing user data through the [ - By default, any supplied user data is pre-pended to the user data supplied by the EKS Managed Node Group service - If users supply an `ami_id`, the service no longers supplies user data to bootstrap nodes; users can enable `enable_bootstrap_user_data` and use the module provided user data template, or provide their own user data template - `bottlerocket` platform user data must be in TOML format + - `windows` platform user data must be in powershell/PS1 script format - Self Managed Node Groups - `linux` platform (default) -> the user data template (bash/shell script) provided by the module is used as the default; users are able to provide their own user data template - `bottlerocket` platform -> the user data template (TOML file) provided by the module is used as the default; users are able to provide their own user data template diff --git a/examples/complete/README.md b/examples/complete/README.md deleted file mode 100644 index 0fd43130da..0000000000 --- a/examples/complete/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# Complete AWS EKS Cluster - -Configuration in this directory creates an AWS EKS cluster with a broad mix of various features and settings provided by this module: - -- AWS EKS cluster -- Disabled EKS cluster -- Self managed node group -- Externally attached self managed node group -- Disabled self managed node group -- EKS managed node group -- Externally attached EKS managed node group -- Disabled self managed node group -- Fargate profile -- Externally attached Fargate profile -- Disabled Fargate profile -- Cluster addons: CoreDNS, Kube-Proxy, and VPC-CNI -- IAM roles for service accounts - -## Usage - -To run this example you need to execute: - -```bash -$ terraform init -$ terraform plan -$ terraform apply -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 4.57 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [disabled\_eks](#module\_disabled\_eks) | ../.. | n/a | -| [disabled\_eks\_managed\_node\_group](#module\_disabled\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | -| [disabled\_fargate\_profile](#module\_disabled\_fargate\_profile) | ../../modules/fargate-profile | n/a | -| [disabled\_self\_managed\_node\_group](#module\_disabled\_self\_managed\_node\_group) | ../../modules/self-managed-node-group | n/a | -| [eks](#module\_eks) | ../.. | n/a | -| [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | -| [fargate\_profile](#module\_fargate\_profile) | ../../modules/fargate-profile | n/a | -| [kms](#module\_kms) | terraform-aws-modules/kms/aws | ~> 1.5 | -| [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ../../modules/self-managed-node-group | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_iam_policy.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | - -## Inputs - -No inputs. - -## Outputs - -| Name | Description | -|------|-------------| -| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | -| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | -| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | -| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | -| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | -| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | -| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | -| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | -| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | -| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | -| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | -| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | -| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | -| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | -| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | -| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | -| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | -| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | -| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | -| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | -| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | -| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | -| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | -| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | -| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | -| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | - diff --git a/examples/complete/main.tf b/examples/complete/main.tf deleted file mode 100644 index 184dba2228..0000000000 --- a/examples/complete/main.tf +++ /dev/null @@ -1,482 +0,0 @@ -provider "aws" { - region = local.region -} - -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - -data "aws_availability_zones" "available" {} -data "aws_caller_identity" "current" {} - -locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - region = "eu-west-1" - - vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_endpoint_public_access = true - - cluster_addons = { - coredns = { - preserve = true - most_recent = true - - timeouts = { - create = "25m" - delete = "10m" - } - } - kube-proxy = { - most_recent = true - } - vpc-cni = { - most_recent = true - } - } - - # External encryption key - create_kms_key = false - cluster_encryption_config = { - resources = ["secrets"] - provider_key_arn = module.kms.key_arn - } - - iam_role_additional_policies = { - additional = aws_iam_policy.additional.arn - } - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - control_plane_subnet_ids = module.vpc.intra_subnets - - # Extend cluster security group rules - cluster_security_group_additional_rules = { - ingress_nodes_ephemeral_ports_tcp = { - description = "Nodes on ephemeral ports" - protocol = "tcp" - from_port = 1025 - to_port = 65535 - type = "ingress" - source_node_security_group = true - } - # Test: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2319 - ingress_source_security_group_id = { - description = "Ingress from another computed security group" - protocol = "tcp" - from_port = 22 - to_port = 22 - type = "ingress" - source_security_group_id = aws_security_group.additional.id - } - } - - # Extend node-to-node security group rules - node_security_group_additional_rules = { - ingress_self_all = { - description = "Node to node all ports/protocols" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "ingress" - self = true - } - # Test: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2319 - ingress_source_security_group_id = { - description = "Ingress from another computed security group" - protocol = "tcp" - from_port = 22 - to_port = 22 - type = "ingress" - source_security_group_id = aws_security_group.additional.id - } - } - - # Self Managed Node Group(s) - self_managed_node_group_defaults = { - vpc_security_group_ids = [aws_security_group.additional.id] - iam_role_additional_policies = { - additional = aws_iam_policy.additional.arn - } - - instance_refresh = { - strategy = "Rolling" - preferences = { - min_healthy_percentage = 66 - } - } - } - - self_managed_node_groups = { - spot = { - instance_type = "m5.large" - instance_market_options = { - market_type = "spot" - } - - pre_bootstrap_user_data = <<-EOT - echo "foo" - export FOO=bar - EOT - - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" - - post_bootstrap_user_data = <<-EOT - cd /tmp - sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm - sudo systemctl enable amazon-ssm-agent - sudo systemctl start amazon-ssm-agent - EOT - } - } - - # EKS Managed Node Group(s) - eks_managed_node_group_defaults = { - ami_type = "AL2_x86_64" - instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] - - attach_cluster_primary_security_group = true - vpc_security_group_ids = [aws_security_group.additional.id] - iam_role_additional_policies = { - additional = aws_iam_policy.additional.arn - } - } - - eks_managed_node_groups = { - blue = {} - green = { - min_size = 1 - max_size = 10 - desired_size = 1 - - instance_types = ["t3.large"] - capacity_type = "SPOT" - labels = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } - - taints = { - dedicated = { - key = "dedicated" - value = "gpuGroup" - effect = "NO_SCHEDULE" - } - } - - block_device_mappings = { - xvda = { - device_name = "/dev/xvda" - ebs = { - volume_size = 100 - volume_type = "gp3" - iops = 3000 - throughput = 150 - delete_on_termination = true - } - } - } - - update_config = { - max_unavailable_percentage = 33 # or set `max_unavailable` - } - - tags = { - ExtraTag = "example" - } - } - } - - # Fargate Profile(s) - fargate_profiles = { - default = { - name = "default" - selectors = [ - { - namespace = "kube-system" - labels = { - k8s-app = "kube-dns" - } - }, - { - namespace = "default" - } - ] - - tags = { - Owner = "test" - } - - timeouts = { - create = "20m" - delete = "20m" - } - } - } - - # Create a new cluster where both an identity provider and Fargate profile is created - # will result in conflicts since only one can take place at a time - # # OIDC Identity provider - # cluster_identity_providers = { - # sts = { - # client_id = "sts.amazonaws.com" - # } - # } - - # aws-auth configmap - manage_aws_auth_configmap = true - - aws_auth_node_iam_role_arns_non_windows = [ - module.eks_managed_node_group.iam_role_arn, - module.self_managed_node_group.iam_role_arn, - ] - aws_auth_fargate_profile_pod_execution_role_arns = [ - module.fargate_profile.fargate_profile_pod_execution_role_arn - ] - - aws_auth_roles = [ - { - rolearn = module.eks_managed_node_group.iam_role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - }, - { - rolearn = module.self_managed_node_group.iam_role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - }, - { - rolearn = module.fargate_profile.fargate_profile_pod_execution_role_arn - username = "system:node:{{SessionName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - "system:node-proxier", - ] - } - ] - - aws_auth_users = [ - { - userarn = "arn:aws:iam::66666666666:user/user1" - username = "user1" - groups = ["system:masters"] - }, - { - userarn = "arn:aws:iam::66666666666:user/user2" - username = "user2" - groups = ["system:masters"] - }, - ] - - aws_auth_accounts = [ - "777777777777", - "888888888888", - ] - - tags = local.tags -} - -################################################################################ -# Sub-Module Usage on Existing/Separate Cluster -################################################################################ - -module "eks_managed_node_group" { - source = "../../modules/eks-managed-node-group" - - name = "separate-eks-mng" - cluster_name = module.eks.cluster_name - cluster_version = module.eks.cluster_version - - subnet_ids = module.vpc.private_subnets - cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id - vpc_security_group_ids = [ - module.eks.cluster_security_group_id, - ] - - ami_type = "BOTTLEROCKET_x86_64" - platform = "bottlerocket" - - # this will get added to what AWS provides - bootstrap_extra_args = <<-EOT - # extra args added - [settings.kernel] - lockdown = "integrity" - - [settings.kubernetes.node-labels] - "label1" = "foo" - "label2" = "bar" - EOT - - tags = merge(local.tags, { Separate = "eks-managed-node-group" }) -} - -module "self_managed_node_group" { - source = "../../modules/self-managed-node-group" - - name = "separate-self-mng" - cluster_name = module.eks.cluster_name - cluster_version = module.eks.cluster_version - cluster_endpoint = module.eks.cluster_endpoint - cluster_auth_base64 = module.eks.cluster_certificate_authority_data - - instance_type = "m5.large" - - subnet_ids = module.vpc.private_subnets - vpc_security_group_ids = [ - module.eks.cluster_primary_security_group_id, - module.eks.cluster_security_group_id, - ] - - tags = merge(local.tags, { Separate = "self-managed-node-group" }) -} - -module "fargate_profile" { - source = "../../modules/fargate-profile" - - name = "separate-fargate-profile" - cluster_name = module.eks.cluster_name - - subnet_ids = module.vpc.private_subnets - selectors = [{ - namespace = "kube-system" - }] - - tags = merge(local.tags, { Separate = "fargate-profile" }) -} - -################################################################################ -# Disabled creation -################################################################################ - -module "disabled_eks" { - source = "../.." - - create = false -} - -module "disabled_fargate_profile" { - source = "../../modules/fargate-profile" - - create = false -} - -module "disabled_eks_managed_node_group" { - source = "../../modules/eks-managed-node-group" - - create = false -} - -module "disabled_self_managed_node_group" { - source = "../../modules/self-managed-node-group" - - create = false -} - -################################################################################ -# Supporting resources -################################################################################ - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 4.0" - - name = local.name - cidr = local.vpc_cidr - - azs = local.azs - private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] - public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] - intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] - - enable_nat_gateway = true - single_nat_gateway = true - - public_subnet_tags = { - "kubernetes.io/role/elb" = 1 - } - - private_subnet_tags = { - "kubernetes.io/role/internal-elb" = 1 - } - - tags = local.tags -} - -resource "aws_security_group" "additional" { - name_prefix = "${local.name}-additional" - vpc_id = module.vpc.vpc_id - - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = [ - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - ] - } - - tags = merge(local.tags, { Name = "${local.name}-additional" }) -} - -resource "aws_iam_policy" "additional" { - name = "${local.name}-additional" - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Action = [ - "ec2:Describe*", - ] - Effect = "Allow" - Resource = "*" - }, - ] - }) -} - -module "kms" { - source = "terraform-aws-modules/kms/aws" - version = "~> 1.5" - - aliases = ["eks/${local.name}"] - description = "${local.name} cluster encryption key" - enable_default_policy = true - key_owners = [data.aws_caller_identity.current.arn] - - tags = local.tags -} diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf deleted file mode 100644 index c1020f3333..0000000000 --- a/examples/complete/outputs.tf +++ /dev/null @@ -1,192 +0,0 @@ -################################################################################ -# Cluster -################################################################################ - -output "cluster_arn" { - description = "The Amazon Resource Name (ARN) of the cluster" - value = module.eks.cluster_arn -} - -output "cluster_certificate_authority_data" { - description = "Base64 encoded certificate data required to communicate with the cluster" - value = module.eks.cluster_certificate_authority_data -} - -output "cluster_endpoint" { - description = "Endpoint for your Kubernetes API server" - value = module.eks.cluster_endpoint -} - -output "cluster_id" { - description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" - value = module.eks.cluster_id -} - -output "cluster_name" { - description = "The name of the EKS cluster" - value = module.eks.cluster_name -} - -output "cluster_oidc_issuer_url" { - description = "The URL on the EKS cluster for the OpenID Connect identity provider" - value = module.eks.cluster_oidc_issuer_url -} - -output "cluster_platform_version" { - description = "Platform version for the cluster" - value = module.eks.cluster_platform_version -} - -output "cluster_status" { - description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" - value = module.eks.cluster_status -} - -output "cluster_security_group_id" { - description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" - value = module.eks.cluster_security_group_id -} - -################################################################################ -# KMS Key -################################################################################ - -output "kms_key_arn" { - description = "The Amazon Resource Name (ARN) of the key" - value = module.eks.kms_key_arn -} - -output "kms_key_id" { - description = "The globally unique identifier for the key" - value = module.eks.kms_key_id -} - -output "kms_key_policy" { - description = "The IAM resource policy set on the key" - value = module.eks.kms_key_policy -} - -################################################################################ -# Security Group -################################################################################ - -output "cluster_security_group_arn" { - description = "Amazon Resource Name (ARN) of the cluster security group" - value = module.eks.cluster_security_group_arn -} - -################################################################################ -# IRSA -################################################################################ - -output "oidc_provider" { - description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" - value = module.eks.oidc_provider -} - -output "oidc_provider_arn" { - description = "The ARN of the OIDC Provider if `enable_irsa = true`" - value = module.eks.oidc_provider_arn -} - -output "cluster_tls_certificate_sha1_fingerprint" { - description = "The SHA1 fingerprint of the public key of the cluster's certificate" - value = module.eks.cluster_tls_certificate_sha1_fingerprint -} - -################################################################################ -# IAM Role -################################################################################ - -output "cluster_iam_role_name" { - description = "IAM role name of the EKS cluster" - value = module.eks.cluster_iam_role_name -} - -output "cluster_iam_role_arn" { - description = "IAM role ARN of the EKS cluster" - value = module.eks.cluster_iam_role_arn -} - -output "cluster_iam_role_unique_id" { - description = "Stable and unique string identifying the IAM role" - value = module.eks.cluster_iam_role_unique_id -} - -################################################################################ -# EKS Addons -################################################################################ - -output "cluster_addons" { - description = "Map of attribute maps for all EKS cluster addons enabled" - value = module.eks.cluster_addons -} - -################################################################################ -# EKS Identity Provider -################################################################################ - -output "cluster_identity_providers" { - description = "Map of attribute maps for all EKS identity providers enabled" - value = module.eks.cluster_identity_providers -} - -################################################################################ -# CloudWatch Log Group -################################################################################ - -output "cloudwatch_log_group_name" { - description = "Name of cloudwatch log group created" - value = module.eks.cloudwatch_log_group_name -} - -output "cloudwatch_log_group_arn" { - description = "Arn of cloudwatch log group created" - value = module.eks.cloudwatch_log_group_arn -} - -################################################################################ -# Fargate Profile -################################################################################ - -output "fargate_profiles" { - description = "Map of attribute maps for all EKS Fargate Profiles created" - value = module.eks.fargate_profiles -} - -################################################################################ -# EKS Managed Node Group -################################################################################ - -output "eks_managed_node_groups" { - description = "Map of attribute maps for all EKS managed node groups created" - value = module.eks.eks_managed_node_groups -} - -output "eks_managed_node_groups_autoscaling_group_names" { - description = "List of the autoscaling group names created by EKS managed node groups" - value = module.eks.eks_managed_node_groups_autoscaling_group_names -} - -################################################################################ -# Self Managed Node Group -################################################################################ - -output "self_managed_node_groups" { - description = "Map of attribute maps for all self managed node groups created" - value = module.eks.self_managed_node_groups -} - -output "self_managed_node_groups_autoscaling_group_names" { - description = "List of the autoscaling group names created by self-managed node groups" - value = module.eks.self_managed_node_groups_autoscaling_group_names -} - -################################################################################ -# Additional -################################################################################ - -output "aws_auth_configmap_yaml" { - description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" - value = module.eks.aws_auth_configmap_yaml -} diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf deleted file mode 100644 index d2ec4a8f9d..0000000000 --- a/examples/complete/versions.tf +++ /dev/null @@ -1,14 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.57" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" - } - } -} diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index c5278e07a0..103d133553 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -29,31 +29,31 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.34 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.57 | +| [aws](#provider\_aws) | >= 5.34 | ## Modules | Name | Source | Version | |------|--------|---------| -| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 1.5 | +| [disabled\_eks](#module\_disabled\_eks) | ../.. | n/a | +| [disabled\_eks\_managed\_node\_group](#module\_disabled\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | +| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 2.1 | | [eks](#module\_eks) | ../.. | n/a | +| [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | -| [vpc\_cni\_irsa](#module\_vpc\_cni\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | ## Resources | Name | Type | |------|------| -| [aws_autoscaling_group_tag.cluster_autoscaler_label_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group_tag) | resource | | [aws_iam_policy.node_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_security_group.remote_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | @@ -70,7 +70,7 @@ No inputs. | Name | Description | |------|-------------| -| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | | [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | | [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index b8097221bc..0d17827fd8 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -2,18 +2,6 @@ provider "aws" { region = local.region } -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - data "aws_caller_identity" "current" {} data "aws_availability_zones" "available" {} @@ -44,14 +32,7 @@ module "eks" { cluster_endpoint_public_access = true # IPV6 - cluster_ip_family = "ipv6" - - # We are using the IRSA created below for permissions - # However, we have to deploy with the policy attached FIRST (when creating a fresh cluster) - # and then turn this off after the cluster/node group is created. Without this initial policy, - # the VPC CNI fails to assign IPs and nodes cannot join the cluster - # See https://github.com/aws/containers-roadmap/issues/1666 for more context - # TODO - remove this policy once AWS releases a managed version similar to AmazonEKS_CNI_Policy (IPv4) + cluster_ip_family = "ipv6" create_cni_ipv6_iam_policy = true cluster_addons = { @@ -62,9 +43,8 @@ module "eks" { most_recent = true } vpc-cni = { - most_recent = true - before_compute = true - service_account_role_arn = module.vpc_cni_irsa.iam_role_arn + most_recent = true + before_compute = true configuration_values = jsonencode({ env = { # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html @@ -79,18 +59,9 @@ module "eks" { subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets - manage_aws_auth_configmap = true - eks_managed_node_group_defaults = { ami_type = "AL2_x86_64" instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] - - # We are using the IRSA created below for permissions - # However, we have to deploy with the policy attached FIRST (when creating a fresh cluster) - # and then turn this off after the cluster/node group is created. Without this initial policy, - # the VPC CNI fails to assign IPs and nodes cannot join the cluster - # See https://github.com/aws/containers-roadmap/issues/1666 for more context - iam_role_attach_cni_policy = true } eks_managed_node_groups = { @@ -264,27 +235,6 @@ module "eks" { additional = aws_iam_policy.node_additional.arn } - schedules = { - scale-up = { - min_size = 2 - max_size = "-1" # Retains current max size - desired_size = 2 - start_time = "2023-03-05T00:00:00Z" - end_time = "2024-03-05T00:00:00Z" - time_zone = "Etc/GMT+0" - recurrence = "0 0 * * *" - }, - scale-down = { - min_size = 0 - max_size = "-1" # Retains current max size - desired_size = 0 - start_time = "2023-03-05T12:00:00Z" - end_time = "2024-03-05T12:00:00Z" - time_zone = "Etc/GMT+0" - recurrence = "0 12 * * *" - } - } - tags = { ExtraTag = "EKS managed node group complete example" } @@ -294,13 +244,59 @@ module "eks" { tags = local.tags } +module "disabled_eks" { + source = "../.." + + create = false +} + +################################################################################ +# Sub-Module Usage on Existing/Separate Cluster +################################################################################ + +module "eks_managed_node_group" { + source = "../../modules/eks-managed-node-group" + + name = "separate-eks-mng" + cluster_name = module.eks.cluster_name + cluster_version = module.eks.cluster_version + + subnet_ids = module.vpc.private_subnets + cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id + vpc_security_group_ids = [ + module.eks.cluster_security_group_id, + ] + + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + + # this will get added to what AWS provides + bootstrap_extra_args = <<-EOT + # extra args added + [settings.kernel] + lockdown = "integrity" + + [settings.kubernetes.node-labels] + "label1" = "foo" + "label2" = "bar" + EOT + + tags = merge(local.tags, { Separate = "eks-managed-node-group" }) +} + +module "disabled_eks_managed_node_group" { + source = "../../modules/eks-managed-node-group" + + create = false +} + ################################################################################ # Supporting Resources ################################################################################ module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 4.0" + version = "~> 5.0" name = local.name cidr = local.vpc_cidr @@ -333,27 +329,9 @@ module "vpc" { tags = local.tags } -module "vpc_cni_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 5.0" - - role_name_prefix = "VPC-CNI-IRSA" - attach_vpc_cni_policy = true - vpc_cni_enable_ipv6 = true - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:aws-node"] - } - } - - tags = local.tags -} - module "ebs_kms_key" { source = "terraform-aws-modules/kms/aws" - version = "~> 1.5" + version = "~> 2.1" description = "Customer managed key to encrypt EKS managed node group volumes" @@ -458,52 +436,3 @@ data "aws_ami" "eks_default_bottlerocket" { values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"] } } - -################################################################################ -# Tags for the ASG to support cluster-autoscaler scale up from 0 -################################################################################ - -locals { - - # We need to lookup K8s taint effect from the AWS API value - taint_effects = { - NO_SCHEDULE = "NoSchedule" - NO_EXECUTE = "NoExecute" - PREFER_NO_SCHEDULE = "PreferNoSchedule" - } - - cluster_autoscaler_label_tags = merge([ - for name, group in module.eks.eks_managed_node_groups : { - for label_name, label_value in coalesce(group.node_group_labels, {}) : "${name}|label|${label_name}" => { - autoscaling_group = group.node_group_autoscaling_group_names[0], - key = "k8s.io/cluster-autoscaler/node-template/label/${label_name}", - value = label_value, - } - } - ]...) - - cluster_autoscaler_taint_tags = merge([ - for name, group in module.eks.eks_managed_node_groups : { - for taint in coalesce(group.node_group_taints, []) : "${name}|taint|${taint.key}" => { - autoscaling_group = group.node_group_autoscaling_group_names[0], - key = "k8s.io/cluster-autoscaler/node-template/taint/${taint.key}" - value = "${taint.value}:${local.taint_effects[taint.effect]}" - } - } - ]...) - - cluster_autoscaler_asg_tags = merge(local.cluster_autoscaler_label_tags, local.cluster_autoscaler_taint_tags) -} - -resource "aws_autoscaling_group_tag" "cluster_autoscaler_label_tags" { - for_each = local.cluster_autoscaler_asg_tags - - autoscaling_group_name = each.value.autoscaling_group - - tag { - key = each.value.key - value = each.value.value - - propagate_at_launch = false - } -} diff --git a/examples/eks_managed_node_group/outputs.tf b/examples/eks_managed_node_group/outputs.tf index 43334ecc0a..0a8873f479 100644 --- a/examples/eks_managed_node_group/outputs.tf +++ b/examples/eks_managed_node_group/outputs.tf @@ -47,6 +47,15 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +################################################################################ +# Access Entry +################################################################################ + +output "access_entries" { + description = "Map of access entries created and their attributes" + value = module.eks.access_entries +} + ################################################################################ # KMS Key ################################################################################ @@ -200,12 +209,3 @@ output "self_managed_node_groups_autoscaling_group_names" { description = "List of the autoscaling group names created by self-managed node groups" value = module.eks.self_managed_node_groups_autoscaling_group_names } - -################################################################################ -# Additional -################################################################################ - -output "aws_auth_configmap_yaml" { - description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" - value = module.eks.aws_auth_configmap_yaml -} diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index d2ec4a8f9d..63e9319d92 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -1,14 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" + version = ">= 5.34" } } } diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 9d41ed40b0..9ea3bf1871 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -19,23 +19,23 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | -| [helm](#requirement\_helm) | >= 2.7 | -| [null](#requirement\_null) | >= 3.0 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.34 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.57 | +| [aws](#provider\_aws) | >= 5.34 | ## Modules | Name | Source | Version | |------|--------|---------| +| [disabled\_fargate\_profile](#module\_disabled\_fargate\_profile) | ../../modules/fargate-profile | n/a | | [eks](#module\_eks) | ../.. | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | +| [fargate\_profile](#module\_fargate\_profile) | ../../modules/fargate-profile | n/a | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | ## Resources @@ -52,7 +52,7 @@ No inputs. | Name | Description | |------|-------------| -| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | | [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | | [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index b67335c94e..0b3c6b46c2 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -6,7 +6,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.27" + cluster_version = "1.29" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -54,59 +54,72 @@ module "eks" { } } - fargate_profiles = merge( - { - example = { - name = "example" - selectors = [ - { - namespace = "backend" - labels = { - Application = "backend" - } - }, - { - namespace = "app-*" - labels = { - Application = "app-wildcard" - } + fargate_profiles = { + example = { + name = "example" + selectors = [ + { + namespace = "backend" + labels = { + Application = "backend" + } + }, + { + namespace = "app-*" + labels = { + Application = "app-wildcard" } - ] - - # Using specific subnets instead of the subnets supplied for the cluster itself - subnet_ids = [module.vpc.private_subnets[1]] - - tags = { - Owner = "secondary" } + ] - timeouts = { - create = "20m" - delete = "20m" - } - } - }, - { for i in range(3) : - "kube-system-${element(split("-", local.azs[i]), 2)}" => { - selectors = [ - { namespace = "kube-system" } - ] - # We want to create a profile per AZ for high availability - subnet_ids = [element(module.vpc.private_subnets, i)] + # Using specific subnets instead of the subnets supplied for the cluster itself + subnet_ids = [module.vpc.private_subnets[1]] + + tags = { + Owner = "secondary" } } - ) + kube-system = { + selectors = [ + { namespace = "kube-system" } + ] + } + } tags = local.tags } +################################################################################ +# Sub-Module Usage on Existing/Separate Cluster +################################################################################ + +module "fargate_profile" { + source = "../../modules/fargate-profile" + + name = "separate-fargate-profile" + cluster_name = module.eks.cluster_name + + subnet_ids = module.vpc.private_subnets + selectors = [{ + namespace = "kube-system" + }] + + tags = merge(local.tags, { Separate = "fargate-profile" }) +} + +module "disabled_fargate_profile" { + source = "../../modules/fargate-profile" + + create = false +} + ################################################################################ # Supporting Resources ################################################################################ module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 4.0" + version = "~> 5.0" name = local.name cidr = local.vpc_cidr diff --git a/examples/fargate_profile/outputs.tf b/examples/fargate_profile/outputs.tf index 43334ecc0a..0a8873f479 100644 --- a/examples/fargate_profile/outputs.tf +++ b/examples/fargate_profile/outputs.tf @@ -47,6 +47,15 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +################################################################################ +# Access Entry +################################################################################ + +output "access_entries" { + description = "Map of access entries created and their attributes" + value = module.eks.access_entries +} + ################################################################################ # KMS Key ################################################################################ @@ -200,12 +209,3 @@ output "self_managed_node_groups_autoscaling_group_names" { description = "List of the autoscaling group names created by self-managed node groups" value = module.eks.self_managed_node_groups_autoscaling_group_names } - -################################################################################ -# Additional -################################################################################ - -output "aws_auth_configmap_yaml" { - description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" - value = module.eks.aws_auth_configmap_yaml -} diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index 10d7878df7..63e9319d92 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -1,18 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.7" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" + version = ">= 5.34" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 06b29e7bb4..b2c42931dd 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -41,6 +41,9 @@ kubectl delete node -l karpenter.sh/provisioner-name=default 2. Remove the resources created by Terraform ```bash +# Necessary to avoid removing Terraform's permissions too soon before its finished +# cleaning up the resources it deployed inside the clsuter +terraform state rm 'module.eks.aws_eks_access_entry.this["cluster_creator_admin"]' || true terraform destroy ``` @@ -51,21 +54,19 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.34 | | [helm](#requirement\_helm) | >= 2.7 | -| [kubectl](#requirement\_kubectl) | >= 1.14 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | -| [null](#requirement\_null) | >= 3.0 | +| [kubectl](#requirement\_kubectl) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.57 | -| [aws.virginia](#provider\_aws.virginia) | >= 4.57 | +| [aws](#provider\_aws) | >= 5.34 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.34 | | [helm](#provider\_helm) | >= 2.7 | -| [kubectl](#provider\_kubectl) | >= 1.14 | +| [kubectl](#provider\_kubectl) | >= 2.0 | ## Modules @@ -73,6 +74,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|--------|---------| | [eks](#module\_eks) | ../.. | n/a | | [karpenter](#module\_karpenter) | ../../modules/karpenter | n/a | +| [karpenter\_disabled](#module\_karpenter\_disabled) | ../../modules/karpenter | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | ## Resources @@ -80,9 +82,9 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_node_class](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_node_class](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | @@ -94,7 +96,7 @@ No inputs. | Name | Description | |------|-------------| -| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | | [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | | [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | @@ -118,19 +120,19 @@ No inputs. | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [karpenter\_event\_rules](#output\_karpenter\_event\_rules) | Map of the event rules created and their attributes | +| [karpenter\_iam\_role\_arn](#output\_karpenter\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the controller IAM role | +| [karpenter\_iam\_role\_name](#output\_karpenter\_iam\_role\_name) | The name of the controller IAM role | +| [karpenter\_iam\_role\_unique\_id](#output\_karpenter\_iam\_role\_unique\_id) | Stable and unique string identifying the controller IAM role | | [karpenter\_instance\_profile\_arn](#output\_karpenter\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | | [karpenter\_instance\_profile\_id](#output\_karpenter\_instance\_profile\_id) | Instance profile's ID | | [karpenter\_instance\_profile\_name](#output\_karpenter\_instance\_profile\_name) | Name of the instance profile | | [karpenter\_instance\_profile\_unique](#output\_karpenter\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | -| [karpenter\_irsa\_arn](#output\_karpenter\_irsa\_arn) | The Amazon Resource Name (ARN) specifying the IAM role for service accounts | -| [karpenter\_irsa\_name](#output\_karpenter\_irsa\_name) | The name of the IAM role for service accounts | -| [karpenter\_irsa\_unique\_id](#output\_karpenter\_irsa\_unique\_id) | Stable and unique string identifying the IAM role for service accounts | +| [karpenter\_node\_iam\_role\_arn](#output\_karpenter\_node\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [karpenter\_node\_iam\_role\_name](#output\_karpenter\_node\_iam\_role\_name) | The name of the IAM role | +| [karpenter\_node\_iam\_role\_unique\_id](#output\_karpenter\_node\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [karpenter\_queue\_arn](#output\_karpenter\_queue\_arn) | The ARN of the SQS queue | | [karpenter\_queue\_name](#output\_karpenter\_queue\_name) | The name of the created Amazon SQS queue | | [karpenter\_queue\_url](#output\_karpenter\_queue\_url) | The URL for the created Amazon SQS queue | -| [karpenter\_role\_arn](#output\_karpenter\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | -| [karpenter\_role\_name](#output\_karpenter\_role\_name) | The name of the IAM role | -| [karpenter\_role\_unique\_id](#output\_karpenter\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index b8d7be97ff..cd36785893 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -7,18 +7,6 @@ provider "aws" { alias = "virginia" } -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - provider "helm" { kubernetes { host = module.eks.cluster_endpoint @@ -54,7 +42,7 @@ data "aws_ecrpublic_authorization_token" "token" { locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.27" + cluster_version = "1.28" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -78,9 +66,11 @@ module "eks" { cluster_version = local.cluster_version cluster_endpoint_public_access = true + # Gives Terraform identity admin access to cluster which will + # allow deploying resources (Karpenter) into the cluster + enable_cluster_creator_admin_permissions = true + cluster_addons = { - kube-proxy = {} - vpc-cni = {} coredns = { configuration_values = jsonencode({ computeType = "Fargate" @@ -106,6 +96,8 @@ module "eks" { } }) } + kube-proxy = {} + vpc-cni = {} } vpc_id = module.vpc.vpc_id @@ -116,19 +108,6 @@ module "eks" { create_cluster_security_group = false create_node_security_group = false - manage_aws_auth_configmap = true - aws_auth_roles = [ - # We need to add in the Karpenter node IAM role for nodes launched by Karpenter - { - rolearn = module.karpenter.role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - }, - ] - fargate_profiles = { karpenter = { selectors = [ @@ -157,41 +136,51 @@ module "eks" { module "karpenter" { source = "../../modules/karpenter" - cluster_name = module.eks.cluster_name - irsa_oidc_provider_arn = module.eks.oidc_provider_arn + cluster_name = module.eks.cluster_name - # In v0.32.0/v1beta1, Karpenter now creates the IAM instance profile - # so we disable the Terraform creation and add the necessary permissions for Karpenter IRSA - enable_karpenter_instance_profile_creation = true + # EKS Fargate currently does not support Pod Identity + enable_irsa = true + irsa_oidc_provider_arn = module.eks.oidc_provider_arn # Used to attach additional IAM policies to the Karpenter node IAM role - iam_role_additional_policies = { + node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } tags = local.tags } -resource "helm_release" "karpenter" { - namespace = "karpenter" - create_namespace = true +module "karpenter_disabled" { + source = "../../modules/karpenter" + + create = false +} +################################################################################ +# Karpenter Helm chart & manifests +# Not required; just to demonstrate functionality of the sub-module +################################################################################ + +resource "helm_release" "karpenter" { + namespace = "karpenter" + create_namespace = true name = "karpenter" repository = "oci://public.ecr.aws/karpenter" repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "v0.32.1" + version = "v0.33.1" + wait = false values = [ <<-EOT settings: clusterName: ${module.eks.cluster_name} clusterEndpoint: ${module.eks.cluster_endpoint} - interruptionQueueName: ${module.karpenter.queue_name} + interruptionQueue: ${module.karpenter.queue_name} serviceAccount: annotations: - eks.amazonaws.com/role-arn: ${module.karpenter.irsa_arn} + eks.amazonaws.com/role-arn: ${module.karpenter.iam_role_arn} EOT ] } @@ -204,7 +193,7 @@ resource "kubectl_manifest" "karpenter_node_class" { name: default spec: amiFamily: AL2 - role: ${module.karpenter.role_name} + role: ${module.karpenter.node_iam_role_name} subnetSelectorTerms: - tags: karpenter.sh/discovery: ${module.eks.cluster_name} diff --git a/examples/karpenter/outputs.tf b/examples/karpenter/outputs.tf index f0ad50bd6a..41fe733f87 100644 --- a/examples/karpenter/outputs.tf +++ b/examples/karpenter/outputs.tf @@ -47,6 +47,15 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +################################################################################ +# Access Entry +################################################################################ + +output "access_entries" { + description = "Map of access entries created and their attributes" + value = module.eks.access_entries +} + ################################################################################ # Security Group ################################################################################ @@ -183,31 +192,22 @@ output "self_managed_node_groups_autoscaling_group_names" { } ################################################################################ -# Additional -################################################################################ - -output "aws_auth_configmap_yaml" { - description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" - value = module.eks.aws_auth_configmap_yaml -} - -################################################################################ -# IAM Role for Service Account (IRSA) +# Karpenter controller IAM Role ################################################################################ -output "karpenter_irsa_name" { - description = "The name of the IAM role for service accounts" - value = module.karpenter.irsa_name +output "karpenter_iam_role_name" { + description = "The name of the controller IAM role" + value = module.karpenter.iam_role_name } -output "karpenter_irsa_arn" { - description = "The Amazon Resource Name (ARN) specifying the IAM role for service accounts" - value = module.karpenter.irsa_arn +output "karpenter_iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the controller IAM role" + value = module.karpenter.iam_role_arn } -output "karpenter_irsa_unique_id" { - description = "Stable and unique string identifying the IAM role for service accounts" - value = module.karpenter.irsa_unique_id +output "karpenter_iam_role_unique_id" { + description = "Stable and unique string identifying the controller IAM role" + value = module.karpenter.iam_role_unique_id } ################################################################################ @@ -242,19 +242,19 @@ output "karpenter_event_rules" { # Node IAM Role ################################################################################ -output "karpenter_role_name" { +output "karpenter_node_iam_role_name" { description = "The name of the IAM role" - value = module.karpenter.role_name + value = module.karpenter.node_iam_role_name } -output "karpenter_role_arn" { +output "karpenter_node_iam_role_arn" { description = "The Amazon Resource Name (ARN) specifying the IAM role" - value = module.karpenter.role_arn + value = module.karpenter.node_iam_role_arn } -output "karpenter_role_unique_id" { +output "karpenter_node_iam_role_unique_id" { description = "Stable and unique string identifying the IAM role" - value = module.karpenter.role_unique_id + value = module.karpenter.node_iam_role_unique_id } ################################################################################ diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index f1f0ee477e..96d7b4ea19 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -1,26 +1,18 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" + version = ">= 5.34" } helm = { source = "hashicorp/helm" version = ">= 2.7" } kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.14" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" + source = "alekc/kubectl" + version = ">= 2.0" } } } diff --git a/examples/outposts/README.md b/examples/outposts/README.md index ff2542c4e9..30fffaf330 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -36,21 +36,28 @@ $ terraform apply Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. +```bash +# Necessary to avoid removing Terraform's permissions too soon before its finished +# cleaning up the resources it deployed inside the clsuter +terraform state rm 'module.eks.aws_eks_access_entry.this["cluster_creator_admin"]' || true +terraform destroy +``` + ## Requirements | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.34 | +| [kubernetes](#requirement\_kubernetes) | >= 2.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.57 | -| [kubernetes](#provider\_kubernetes) | >= 2.10 | +| [aws](#provider\_aws) | >= 5.34 | +| [kubernetes](#provider\_kubernetes) | >= 2.20 | ## Modules @@ -80,7 +87,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Description | |------|-------------| -| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | | [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | | [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | diff --git a/examples/outposts/main.tf b/examples/outposts/main.tf index fc50b53c89..4b13f52465 100644 --- a/examples/outposts/main.tf +++ b/examples/outposts/main.tf @@ -2,21 +2,9 @@ provider "aws" { region = var.region } -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # Note: `cluster_id` is used with Outposts for auth - args = ["eks", "get-token", "--cluster-id", module.eks.cluster_id, "--region", var.region] - } -} - locals { name = "ex-${basename(path.cwd)}" - cluster_version = "1.27" # Required by EKS on Outposts + cluster_version = "1.29" outpost_arn = element(tolist(data.aws_outposts_outposts.this.arns), 0) instance_type = element(tolist(data.aws_outposts_outpost_instance_types.this.instance_types), 0) @@ -41,6 +29,10 @@ module "eks" { cluster_endpoint_public_access = false # Not available on Outpost cluster_endpoint_private_access = true + # Gives Terraform identity admin access to cluster which will + # allow deploying resources (EBS storage class) into the cluster + enable_cluster_creator_admin_permissions = true + vpc_id = data.aws_vpc.this.id subnet_ids = data.aws_subnets.this.ids @@ -49,9 +41,6 @@ module "eks" { outpost_arns = [local.outpost_arn] } - # Local clusters will automatically add the node group IAM role to the aws-auth configmap - manage_aws_auth_configmap = true - # Extend cluster security group rules cluster_security_group_additional_rules = { ingress_vpc_https = { diff --git a/examples/outposts/outputs.tf b/examples/outposts/outputs.tf index 43334ecc0a..0a8873f479 100644 --- a/examples/outposts/outputs.tf +++ b/examples/outposts/outputs.tf @@ -47,6 +47,15 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +################################################################################ +# Access Entry +################################################################################ + +output "access_entries" { + description = "Map of access entries created and their attributes" + value = module.eks.access_entries +} + ################################################################################ # KMS Key ################################################################################ @@ -200,12 +209,3 @@ output "self_managed_node_groups_autoscaling_group_names" { description = "List of the autoscaling group names created by self-managed node groups" value = module.eks.self_managed_node_groups_autoscaling_group_names } - -################################################################################ -# Additional -################################################################################ - -output "aws_auth_configmap_yaml" { - description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" - value = module.eks.aws_auth_configmap_yaml -} diff --git a/examples/outposts/prerequisites/main.tf b/examples/outposts/prerequisites/main.tf index 014418121d..66ab2a4e29 100644 --- a/examples/outposts/prerequisites/main.tf +++ b/examples/outposts/prerequisites/main.tf @@ -23,7 +23,7 @@ locals { module "ssm_bastion_ec2" { source = "terraform-aws-modules/ec2-instance/aws" - version = "~> 4.2" + version = "~> 5.5" name = "${local.name}-bastion" @@ -56,7 +56,7 @@ module "ssm_bastion_ec2" { rm terraform_${local.terraform_version}_linux_amd64.zip 2> /dev/null # Install kubectl - curl -LO https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubectl + curl -LO https://dl.k8s.io/release/v1.29.0/bin/linux/amd64/kubectl install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl # Remove default awscli which is v1 - we want latest v2 @@ -80,7 +80,7 @@ module "ssm_bastion_ec2" { module "bastion_security_group" { source = "terraform-aws-modules/security-group/aws" - version = "~> 4.0" + version = "~> 5.0" name = "${local.name}-bastion" description = "Security group to allow provisioning ${local.name} EKS local cluster on Outposts" diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf index 01d187af62..63e9319d92 100644 --- a/examples/outposts/prerequisites/versions.tf +++ b/examples/outposts/prerequisites/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" + version = ">= 5.34" } } } diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf index d2ec4a8f9d..dff26f6939 100644 --- a/examples/outposts/versions.tf +++ b/examples/outposts/versions.tf @@ -1,14 +1,14 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" + version = ">= 5.34" } kubernetes = { source = "hashicorp/kubernetes" - version = ">= 2.10" + version = ">= 2.20" } } } diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 5d849be0b9..c19999d9b9 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -25,24 +25,25 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.34 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.57 | +| [aws](#provider\_aws) | >= 5.34 | ## Modules | Name | Source | Version | |------|--------|---------| -| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 1.5 | +| [disabled\_self\_managed\_node\_group](#module\_disabled\_self\_managed\_node\_group) | ../../modules/self-managed-node-group | n/a | +| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 2.0 | | [eks](#module\_eks) | ../.. | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | ~> 2.1 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | ## Resources @@ -62,7 +63,7 @@ No inputs. | Name | Description | |------|-------------| -| [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles | +| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | | [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | | [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index d359226f3c..dc125e1fbb 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -2,24 +2,12 @@ provider "aws" { region = local.region } -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - data "aws_caller_identity" "current" {} data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.27" + cluster_version = "1.29" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -59,9 +47,12 @@ module "eks" { subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets - # Self managed node groups will not automatically create the aws-auth configmap so we need to - create_aws_auth_configmap = true - manage_aws_auth_configmap = true + # External encryption key + create_kms_key = false + cluster_encryption_config = { + resources = ["secrets"] + provider_key_arn = module.kms.key_arn + } self_managed_node_group_defaults = { # enable discovery of autoscaling groups by cluster-autoscaler @@ -141,36 +132,6 @@ module "eks" { } } - efa = { - min_size = 1 - max_size = 2 - desired_size = 1 - - # aws ec2 describe-instance-types --region eu-west-1 --filters Name=network-info.efa-supported,Values=true --query "InstanceTypes[*].[InstanceType]" --output text | sort - instance_type = "c5n.9xlarge" - - post_bootstrap_user_data = <<-EOT - # Install EFA - curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz - tar -xf aws-efa-installer-latest.tar.gz && cd aws-efa-installer - ./efa_installer.sh -y --minimal - fi_info -p efa -t FI_EP_RDM - - # Disable ptrace - sysctl -w kernel.yama.ptrace_scope=0 - EOT - - network_interfaces = [ - { - description = "EFA interface example" - delete_on_termination = true - device_index = 0 - associate_public_ip_address = false - interface_type = "efa" - } - ] - } - # Complete complete = { name = "complete-self-mng" @@ -287,12 +248,6 @@ module "eks" { additional = aws_iam_policy.additional.arn } - timeouts = { - create = "80m" - update = "80m" - delete = "80m" - } - tags = { ExtraTag = "Self managed node group complete example" } @@ -302,13 +257,19 @@ module "eks" { tags = local.tags } +module "disabled_self_managed_node_group" { + source = "../../modules/self-managed-node-group" + + create = false +} + ################################################################################ # Supporting Resources ################################################################################ module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 4.0" + version = "~> 5.0" name = local.name cidr = local.vpc_cidr @@ -364,7 +325,7 @@ module "key_pair" { module "ebs_kms_key" { source = "terraform-aws-modules/kms/aws" - version = "~> 1.5" + version = "~> 2.0" description = "Customer managed key to encrypt EKS managed node group volumes" @@ -386,6 +347,18 @@ module "ebs_kms_key" { tags = local.tags } +module "kms" { + source = "terraform-aws-modules/kms/aws" + version = "~> 2.1" + + aliases = ["eks/${local.name}"] + description = "${local.name} cluster encryption key" + enable_default_policy = true + key_owners = [data.aws_caller_identity.current.arn] + + tags = local.tags +} + resource "aws_iam_policy" "additional" { name = "${local.name}-additional" description = "Example usage of node additional policy" diff --git a/examples/self_managed_node_group/outputs.tf b/examples/self_managed_node_group/outputs.tf index 43334ecc0a..0a8873f479 100644 --- a/examples/self_managed_node_group/outputs.tf +++ b/examples/self_managed_node_group/outputs.tf @@ -47,6 +47,15 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +################################################################################ +# Access Entry +################################################################################ + +output "access_entries" { + description = "Map of access entries created and their attributes" + value = module.eks.access_entries +} + ################################################################################ # KMS Key ################################################################################ @@ -200,12 +209,3 @@ output "self_managed_node_groups_autoscaling_group_names" { description = "List of the autoscaling group names created by self-managed node groups" value = module.eks.self_managed_node_groups_autoscaling_group_names } - -################################################################################ -# Additional -################################################################################ - -output "aws_auth_configmap_yaml" { - description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" - value = module.eks.aws_auth_configmap_yaml -} diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index d2ec4a8f9d..63e9319d92 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -1,14 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" + version = ">= 5.34" } } } diff --git a/examples/user_data/README.md b/examples/user_data/README.md index cea7dce755..b2215a02bb 100644 --- a/examples/user_data/README.md +++ b/examples/user_data/README.md @@ -17,7 +17,7 @@ $ terraform apply | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | +| [terraform](#requirement\_terraform) | >= 1.3 | ## Providers @@ -35,6 +35,10 @@ No providers. | [eks\_mng\_linux\_custom\_ami](#module\_eks\_mng\_linux\_custom\_ami) | ../../modules/_user_data | n/a | | [eks\_mng\_linux\_custom\_template](#module\_eks\_mng\_linux\_custom\_template) | ../../modules/_user_data | n/a | | [eks\_mng\_linux\_no\_op](#module\_eks\_mng\_linux\_no\_op) | ../../modules/_user_data | n/a | +| [eks\_mng\_windows\_additional](#module\_eks\_mng\_windows\_additional) | ../../modules/_user_data | n/a | +| [eks\_mng\_windows\_custom\_ami](#module\_eks\_mng\_windows\_custom\_ami) | ../../modules/_user_data | n/a | +| [eks\_mng\_windows\_custom\_template](#module\_eks\_mng\_windows\_custom\_template) | ../../modules/_user_data | n/a | +| [eks\_mng\_windows\_no\_op](#module\_eks\_mng\_windows\_no\_op) | ../../modules/_user_data | n/a | | [self\_mng\_bottlerocket\_bootstrap](#module\_self\_mng\_bottlerocket\_bootstrap) | ../../modules/_user_data | n/a | | [self\_mng\_bottlerocket\_custom\_template](#module\_self\_mng\_bottlerocket\_custom\_template) | ../../modules/_user_data | n/a | | [self\_mng\_bottlerocket\_no\_op](#module\_self\_mng\_bottlerocket\_no\_op) | ../../modules/_user_data | n/a | @@ -65,6 +69,10 @@ No inputs. | [eks\_mng\_linux\_custom\_ami](#output\_eks\_mng\_linux\_custom\_ami) | Base64 decoded user data rendered for the provided inputs | | [eks\_mng\_linux\_custom\_template](#output\_eks\_mng\_linux\_custom\_template) | Base64 decoded user data rendered for the provided inputs | | [eks\_mng\_linux\_no\_op](#output\_eks\_mng\_linux\_no\_op) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_windows\_additional](#output\_eks\_mng\_windows\_additional) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_windows\_custom\_ami](#output\_eks\_mng\_windows\_custom\_ami) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_windows\_custom\_template](#output\_eks\_mng\_windows\_custom\_template) | Base64 decoded user data rendered for the provided inputs | +| [eks\_mng\_windows\_no\_op](#output\_eks\_mng\_windows\_no\_op) | Base64 decoded user data rendered for the provided inputs | | [self\_mng\_bottlerocket\_bootstrap](#output\_self\_mng\_bottlerocket\_bootstrap) | Base64 decoded user data rendered for the provided inputs | | [self\_mng\_bottlerocket\_custom\_template](#output\_self\_mng\_bottlerocket\_custom\_template) | Base64 decoded user data rendered for the provided inputs | | [self\_mng\_bottlerocket\_no\_op](#output\_self\_mng\_bottlerocket\_no\_op) | Base64 decoded user data rendered for the provided inputs | diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index d7d513190e..ea08c8c6d0 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -121,6 +121,69 @@ module "eks_mng_bottlerocket_custom_template" { EOT } +# EKS managed node group - windows +module "eks_mng_windows_no_op" { + source = "../../modules/_user_data" + + platform = "windows" +} + +module "eks_mng_windows_additional" { + source = "../../modules/_user_data" + + platform = "windows" + + pre_bootstrap_user_data = <<-EOT + [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + EOT +} + +module "eks_mng_windows_custom_ami" { + source = "../../modules/_user_data" + + platform = "windows" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + enable_bootstrap_user_data = true + + pre_bootstrap_user_data = <<-EOT + [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + EOT + # I don't know if this is the right way on Windows, but its just a string check here anyways + bootstrap_extra_args = "-KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot" + + post_bootstrap_user_data = <<-EOT + [string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + EOT +} + +module "eks_mng_windows_custom_template" { + source = "../../modules/_user_data" + + platform = "windows" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + enable_bootstrap_user_data = true + + user_data_template_path = "${path.module}/templates/windows_custom.tpl" + + pre_bootstrap_user_data = <<-EOT + [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + EOT + # I don't know if this is the right way on Windows, but its just a string check here anyways + bootstrap_extra_args = "-KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot" + + post_bootstrap_user_data = <<-EOT + [string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + EOT +} + # Self managed node group - linux module "self_mng_linux_no_op" { source = "../../modules/_user_data" @@ -247,7 +310,7 @@ module "self_mng_windows_bootstrap" { pre_bootstrap_user_data = <<-EOT [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' EOT - # I don't know if this is the right way on WindowsOS, but its just a string check here anyways + # I don't know if this is the right way on Windows, but its just a string check here anyways bootstrap_extra_args = "-KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot" post_bootstrap_user_data = <<-EOT @@ -272,7 +335,7 @@ module "self_mng_windows_custom_template" { pre_bootstrap_user_data = <<-EOT [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' EOT - # I don't know if this is the right way on WindowsOS, but its just a string check here anyways + # I don't know if this is the right way on Windows, but its just a string check here anyways bootstrap_extra_args = "-KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot" post_bootstrap_user_data = <<-EOT diff --git a/examples/user_data/outputs.tf b/examples/user_data/outputs.tf index dd2c3407e1..768d81c37a 100644 --- a/examples/user_data/outputs.tf +++ b/examples/user_data/outputs.tf @@ -40,6 +40,27 @@ output "eks_mng_bottlerocket_custom_template" { value = base64decode(module.eks_mng_bottlerocket_custom_template.user_data) } +# EKS managed node group - windows +output "eks_mng_windows_no_op" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_windows_no_op.user_data) +} + +output "eks_mng_windows_additional" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_windows_additional.user_data) +} + +output "eks_mng_windows_custom_ami" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_windows_custom_ami.user_data) +} + +output "eks_mng_windows_custom_template" { + description = "Base64 decoded user data rendered for the provided inputs" + value = base64decode(module.eks_mng_windows_custom_template.user_data) +} + # Self managed node group - linux output "self_mng_linux_no_op" { description = "Base64 decoded user data rendered for the provided inputs" diff --git a/examples/user_data/versions.tf b/examples/user_data/versions.tf index 7117131f4c..696426845c 100644 --- a/examples/user_data/versions.tf +++ b/examples/user_data/versions.tf @@ -1,3 +1,3 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" } diff --git a/main.tf b/main.tf index 6a25f3a63f..a934c0f38e 100644 --- a/main.tf +++ b/main.tf @@ -12,6 +12,8 @@ data "aws_iam_session_context" "current" { locals { create = var.create && var.putin_khuylo + partition = data.aws_partition.current.partition + cluster_role = try(aws_iam_role.this[0].arn, var.iam_role_arn) create_outposts_local_cluster = length(var.outpost_config) > 0 @@ -30,6 +32,17 @@ resource "aws_eks_cluster" "this" { version = var.cluster_version enabled_cluster_log_types = var.cluster_enabled_log_types + access_config { + authentication_mode = var.authentication_mode + + # See access entries below - this is a one time operation from the EKS API. + # Instead, we are hardcoding this to false and if users wish to achieve this + # same functionality, we will do that through an access entry which can be + # enabled or disabled at any time of their choosing using the variable + # var.enable_cluster_creator_admin_permissions + bootstrap_cluster_creator_admin_permissions = false + } + vpc_config { security_group_ids = compact(distinct(concat(var.cluster_additional_security_group_ids, [local.cluster_security_group_id]))) subnet_ids = coalescelist(var.control_plane_subnet_ids, var.subnet_ids) @@ -71,14 +84,15 @@ resource "aws_eks_cluster" "this" { } tags = merge( + { terraform-aws-modules = "eks" }, var.tags, var.cluster_tags, ) timeouts { - create = lookup(var.cluster_timeouts, "create", null) - update = lookup(var.cluster_timeouts, "update", null) - delete = lookup(var.cluster_timeouts, "delete", null) + create = try(var.cluster_timeouts.create, null) + update = try(var.cluster_timeouts.update, null) + delete = try(var.cluster_timeouts.delete, null) } depends_on = [ @@ -109,6 +123,7 @@ resource "aws_cloudwatch_log_group" "this" { name = "/aws/eks/${var.cluster_name}/cluster" retention_in_days = var.cloudwatch_log_group_retention_in_days kms_key_id = var.cloudwatch_log_group_kms_key_id + log_group_class = var.cloudwatch_log_group_class tags = merge( var.tags, @@ -117,6 +132,92 @@ resource "aws_cloudwatch_log_group" "this" { ) } +################################################################################ +# Access Entry +################################################################################ + +locals { + # This replaces the one time logic from the EKS API with something that can be + # better controlled by users through Terraform + bootstrap_cluster_creator_admin_permissions = { + cluster_creator = { + principal_arn = data.aws_iam_session_context.current.issuer_arn + type = "STANDARD" + + policy_associations = { + admin = { + policy_arn = "arn:${local.partition}:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" + access_scope = { + type = "cluster" + } + } + } + } + } + + # Merge the bootstrap behavior with the entries that users provide + merged_access_entries = merge( + { for k, v in local.bootstrap_cluster_creator_admin_permissions : k => v if var.enable_cluster_creator_admin_permissions }, + var.access_entries, + ) + + # Flatten out entries and policy associations so users can specify the policy + # associations within a single entry + flattened_access_entries = flatten([ + for entry_key, entry_val in local.merged_access_entries : [ + for pol_key, pol_val in lookup(entry_val, "policy_associations", {}) : + merge( + { + principal_arn = entry_val.principal_arn + kubernetes_groups = lookup(entry_val, "kubernetes_groups", []) + tags = lookup(entry_val, "tags", {}) + type = lookup(entry_val, "type", "STANDARD") + user_name = lookup(entry_val, "user_name", null) + }, + { for k, v in { + association_policy_arn = pol_val.policy_arn + association_access_scope_type = pol_val.access_scope.type + association_access_scope_namespaces = lookup(pol_val.access_scope, "namespaces", []) + } : k => v if !contains(["EC2_LINUX", "EC2_WINDOWS", "FARGATE_LINUX"], lookup(entry_val, "type", "STANDARD")) }, + { + entry_key = entry_key + pol_key = pol_key + } + ) + ] + ]) +} + +resource "aws_eks_access_entry" "this" { + for_each = { for k, v in local.flattened_access_entries : "${v.entry_key}_${v.pol_key}" => v if local.create } + + cluster_name = aws_eks_cluster.this[0].name + kubernetes_groups = try(each.value.kubernetes_groups, []) + principal_arn = each.value.principal_arn + type = try(each.value.type, "STANDARD") + user_name = try(each.value.user_name, null) + + tags = merge(var.tags, try(each.value.tags, {})) +} + +resource "aws_eks_access_policy_association" "this" { + for_each = { for k, v in local.flattened_access_entries : "${v.entry_key}_${v.pol_key}" => v if local.create } + + access_scope { + namespaces = try(each.value.association_access_scope_namespaces, []) + type = each.value.association_access_scope_type + } + + cluster_name = aws_eks_cluster.this[0].name + + policy_arn = each.value.association_policy_arn + principal_arn = each.value.principal_arn + + depends_on = [ + aws_eks_access_entry.this, + ] +} + ################################################################################ # KMS Key ################################################################################ @@ -148,7 +249,10 @@ module "kms" { cluster = { name = "eks/${var.cluster_name}" } } - tags = var.tags + tags = merge( + { terraform-aws-modules = "eks" }, + var.tags, + ) } ################################################################################ @@ -239,7 +343,7 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { # Not available on outposts count = local.create_oidc_provider ? 1 : 0 - client_id_list = distinct(compact(concat(["sts.${local.dns_suffix}"], var.openid_connect_audiences))) + client_id_list = distinct(compact(concat(["sts.amazonaws.com"], var.openid_connect_audiences))) thumbprint_list = concat(local.oidc_root_ca_thumbprint, var.custom_oidc_thumbprints) url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer @@ -256,13 +360,9 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { locals { create_iam_role = local.create && var.create_iam_role iam_role_name = coalesce(var.iam_role_name, "${var.cluster_name}-cluster") - iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" cluster_encryption_policy_name = coalesce(var.cluster_encryption_policy_name, "${local.iam_role_name}-ClusterEncryption") - - # TODO - hopefully this can be removed once the AWS endpoint is named properly in China - # https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1904 - dns_suffix = coalesce(var.cluster_iam_role_dns_suffix, data.aws_partition.current.dns_suffix) } data "aws_iam_policy_document" "assume_role_policy" { @@ -274,7 +374,7 @@ data "aws_iam_policy_document" "assume_role_policy" { principals { type = "Service" - identifiers = ["eks.${local.dns_suffix}"] + identifiers = ["eks.amazonaws.com"] } dynamic "principals" { @@ -283,7 +383,7 @@ data "aws_iam_policy_document" "assume_role_policy" { content { type = "Service" identifiers = [ - "ec2.${local.dns_suffix}", + "ec2.amazonaws.com", ] } } @@ -394,11 +494,12 @@ resource "aws_eks_addon" "this" { cluster_name = aws_eks_cluster.this[0].name addon_name = try(each.value.name, each.key) - addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) - configuration_values = try(each.value.configuration_values, null) - preserve = try(each.value.preserve, null) - resolve_conflicts = try(each.value.resolve_conflicts, "OVERWRITE") - service_account_role_arn = try(each.value.service_account_role_arn, null) + addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) + configuration_values = try(each.value.configuration_values, null) + preserve = try(each.value.preserve, true) + resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, "OVERWRITE") + resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE") + service_account_role_arn = try(each.value.service_account_role_arn, null) timeouts { create = try(each.value.timeouts.create, var.cluster_addons_timeouts.create, null) @@ -422,11 +523,12 @@ resource "aws_eks_addon" "before_compute" { cluster_name = aws_eks_cluster.this[0].name addon_name = try(each.value.name, each.key) - addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) - configuration_values = try(each.value.configuration_values, null) - preserve = try(each.value.preserve, null) - resolve_conflicts = try(each.value.resolve_conflicts, "OVERWRITE") - service_account_role_arn = try(each.value.service_account_role_arn, null) + addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) + configuration_values = try(each.value.configuration_values, null) + preserve = try(each.value.preserve, true) + resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, "OVERWRITE") + resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE") + service_account_role_arn = try(each.value.service_account_role_arn, null) timeouts { create = try(each.value.timeouts.create, var.cluster_addons_timeouts.create, null) @@ -468,111 +570,3 @@ resource "aws_eks_identity_provider_config" "this" { tags = var.tags } - -################################################################################ -# aws-auth configmap -################################################################################ - -locals { - node_iam_role_arns_non_windows = distinct( - compact( - concat( - [for group in module.eks_managed_node_group : group.iam_role_arn if group.platform != "windows"], - [for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"], - var.aws_auth_node_iam_role_arns_non_windows, - ) - ) - ) - - node_iam_role_arns_windows = distinct( - compact( - concat( - [for group in module.eks_managed_node_group : group.iam_role_arn if group.platform == "windows"], - [for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"], - var.aws_auth_node_iam_role_arns_windows, - ) - ) - ) - - fargate_profile_pod_execution_role_arns = distinct( - compact( - concat( - [for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn], - var.aws_auth_fargate_profile_pod_execution_role_arns, - ) - ) - ) - - aws_auth_configmap_data = { - mapRoles = yamlencode(concat( - [for role_arn in local.node_iam_role_arns_non_windows : { - rolearn = role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - } - ], - [for role_arn in local.node_iam_role_arns_windows : { - rolearn = role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "eks:kube-proxy-windows", - "system:bootstrappers", - "system:nodes", - ] - } - ], - # Fargate profile - [for role_arn in local.fargate_profile_pod_execution_role_arns : { - rolearn = role_arn - username = "system:node:{{SessionName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - "system:node-proxier", - ] - } - ], - var.aws_auth_roles - )) - mapUsers = yamlencode(var.aws_auth_users) - mapAccounts = yamlencode(var.aws_auth_accounts) - } -} - -resource "kubernetes_config_map" "aws_auth" { - count = var.create && var.create_aws_auth_configmap ? 1 : 0 - - metadata { - name = "aws-auth" - namespace = "kube-system" - } - - data = local.aws_auth_configmap_data - - lifecycle { - # We are ignoring the data here since we will manage it with the resource below - # This is only intended to be used in scenarios where the configmap does not exist - ignore_changes = [data, metadata[0].labels, metadata[0].annotations] - } -} - -resource "kubernetes_config_map_v1_data" "aws_auth" { - count = var.create && var.manage_aws_auth_configmap ? 1 : 0 - - force = true - - metadata { - name = "aws-auth" - namespace = "kube-system" - } - - data = local.aws_auth_configmap_data - - depends_on = [ - # Required for instances where the configmap does not exist yet to avoid race condition - kubernetes_config_map.aws_auth, - ] -} diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 0853fd9e1a..4b50bfe35b 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -9,7 +9,7 @@ See [`examples/user_data/`](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | +| [terraform](#requirement\_terraform) | >= 1.3 | | [cloudinit](#requirement\_cloudinit) | >= 2.0 | ## Providers diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index 8ace10539d..5d9e669538 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -38,9 +38,11 @@ locals { } windows = { - user_data = var.create && var.platform == "windows" && var.enable_bootstrap_user_data ? base64encode(templatefile( + user_data = var.create && var.platform == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? base64encode(templatefile( coalesce(var.user_data_template_path, "${path.module}/../../templates/windows_user_data.tpl"), { + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami + enable_bootstrap_user_data = var.enable_bootstrap_user_data # Required to bootstrap node cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint diff --git a/modules/_user_data/versions.tf b/modules/_user_data/versions.tf index 2dbd12cdc0..00dcffe370 100644 --- a/modules/_user_data/versions.tf +++ b/modules/_user_data/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { cloudinit = { diff --git a/modules/aws-auth/README.md b/modules/aws-auth/README.md new file mode 100644 index 0000000000..d66aa9fca7 --- /dev/null +++ b/modules/aws-auth/README.md @@ -0,0 +1,81 @@ +# `aws-auth` Module + +Configuration in this directory creates/updates the `aws-auth` ConfigMap. + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws//modules/aws-auth" + version = "~> 20.0" + + manage_aws_auth_configmap = true + + aws_auth_roles = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] + + aws_auth_users = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] + + aws_auth_accounts = [ + "777777777777", + "888888888888", + ] +} +``` + +## Usage + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [kubernetes](#requirement\_kubernetes) | >= 2.20 | + +## Providers + +| Name | Version | +|------|---------| +| [kubernetes](#provider\_kubernetes) | >= 2.20 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | +| [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_auth\_accounts](#input\_aws\_auth\_accounts) | List of account maps to add to the aws-auth configmap | `list(any)` | `[]` | no | +| [aws\_auth\_roles](#input\_aws\_auth\_roles) | List of role maps to add to the aws-auth configmap | `list(any)` | `[]` | no | +| [aws\_auth\_users](#input\_aws\_auth\_users) | List of user maps to add to the aws-auth configmap | `list(any)` | `[]` | no | +| [create](#input\_create) | Controls if resources should be created (affects all resources) | `bool` | `true` | no | +| [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | +| [manage\_aws\_auth\_configmap](#input\_manage\_aws\_auth\_configmap) | Determines whether to manage the aws-auth configmap | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/modules/aws-auth/main.tf b/modules/aws-auth/main.tf new file mode 100644 index 0000000000..2f7e9694a7 --- /dev/null +++ b/modules/aws-auth/main.tf @@ -0,0 +1,47 @@ + +################################################################################ +# aws-auth configmap +################################################################################ + +locals { + aws_auth_configmap_data = { + mapRoles = yamlencode(var.aws_auth_roles) + mapUsers = yamlencode(var.aws_auth_users) + mapAccounts = yamlencode(var.aws_auth_accounts) + } +} + +resource "kubernetes_config_map" "aws_auth" { + count = var.create && var.create_aws_auth_configmap ? 1 : 0 + + metadata { + name = "aws-auth" + namespace = "kube-system" + } + + data = local.aws_auth_configmap_data + + lifecycle { + # We are ignoring the data here since we will manage it with the resource below + # This is only intended to be used in scenarios where the configmap does not exist + ignore_changes = [data, metadata[0].labels, metadata[0].annotations] + } +} + +resource "kubernetes_config_map_v1_data" "aws_auth" { + count = var.create && var.manage_aws_auth_configmap ? 1 : 0 + + force = true + + metadata { + name = "aws-auth" + namespace = "kube-system" + } + + data = local.aws_auth_configmap_data + + depends_on = [ + # Required for instances where the configmap does not exist yet to avoid race condition + kubernetes_config_map.aws_auth, + ] +} diff --git a/examples/complete/variables.tf b/modules/aws-auth/outputs.tf similarity index 100% rename from examples/complete/variables.tf rename to modules/aws-auth/outputs.tf diff --git a/modules/aws-auth/variables.tf b/modules/aws-auth/variables.tf new file mode 100644 index 0000000000..3aaeb023e3 --- /dev/null +++ b/modules/aws-auth/variables.tf @@ -0,0 +1,39 @@ +variable "create" { + description = "Controls if resources should be created (affects all resources)" + type = bool + default = true +} + +################################################################################ +# aws-auth ConfigMap +################################################################################ + +variable "create_aws_auth_configmap" { + description = "Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap`" + type = bool + default = false +} + +variable "manage_aws_auth_configmap" { + description = "Determines whether to manage the aws-auth configmap" + type = bool + default = true +} + +variable "aws_auth_roles" { + description = "List of role maps to add to the aws-auth configmap" + type = list(any) + default = [] +} + +variable "aws_auth_users" { + description = "List of user maps to add to the aws-auth configmap" + type = list(any) + default = [] +} + +variable "aws_auth_accounts" { + description = "List of account maps to add to the aws-auth configmap" + type = list(any) + default = [] +} diff --git a/modules/aws-auth/versions.tf b/modules/aws-auth/versions.tf new file mode 100644 index 0000000000..8ddb07fa57 --- /dev/null +++ b/modules/aws-auth/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3" + + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.20" + } + } +} diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 69d33ae51e..ebae013b92 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -63,14 +63,14 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.34 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.57 | +| [aws](#provider\_aws) | >= 5.34 | ## Modules @@ -98,7 +98,7 @@ module "eks_managed_node_group" { |------|-------------|------|---------|:--------:| | [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance. If not supplied, EKS will use its own default image | `string` | `""` | no | | [ami\_release\_version](#input\_ami\_release\_version) | AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version | `string` | `null` | no | -| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Valid values are `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `CUSTOM`, `BOTTLEROCKET_ARM_64`, `BOTTLEROCKET_x86_64` | `string` | `null` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `null` | no | | [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index e4a59758b4..16ca010ae6 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -409,7 +409,7 @@ data "aws_iam_policy_document" "assume_role_policy" { principals { type = "Service" - identifiers = ["ec2.${data.aws_partition.current.dns_suffix}"] + identifiers = ["ec2.amazonaws.com"] } } } diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 197cd28c95..ede9dc4c50 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -321,7 +321,7 @@ variable "use_name_prefix" { } variable "ami_type" { - description = "Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Valid values are `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `CUSTOM`, `BOTTLEROCKET_ARM_64`, `BOTTLEROCKET_x86_64`" + description = "Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values" type = string default = null } diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 01d187af62..63e9319d92 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" + version = ">= 5.34" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index d841524ac4..8656a6f191 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -28,14 +28,14 @@ module "fargate_profile" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.34 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.57 | +| [aws](#provider\_aws) | >= 5.34 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 01d187af62..63e9319d92 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" + version = ">= 5.34" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 7b813471ad..3262f71c63 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -7,30 +7,15 @@ Configuration in this directory creates the AWS resources required by Karpenter ### All Resources (Default) In the following example, the Karpenter module will create: -- An IAM role for service accounts (IRSA) with a narrowly scoped IAM policy for the Karpenter controller to utilize -- An IAM role and instance profile for the nodes created by Karpenter to utilize - - Note: This IAM role ARN will need to be added to the `aws-auth` configmap for nodes to join the cluster successfully -- An SQS queue and Eventbridge event rules for Karpenter to utilize for spot termination handling, capacity rebalancing, etc. - -This setup is great for running Karpenter on EKS Fargate: +- An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller +- A Node IAM role that Karpenter will use to create an Instance Profile for the nodes to receive IAM permissions +- An access entry for the Node IAM role to allow nodes to join the cluster +- SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. ```hcl module "eks" { - source = "terraform-aws-modules/eks" + source = "terraform-aws-modules/eks/aws" - # Shown just for connection between cluster and Karpenter sub-module below - manage_aws_auth_configmap = true - aws_auth_roles = [ - # We need to add in the Karpenter node IAM role for nodes launched by Karpenter - { - rolearn = module.karpenter.role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - }, - ] ... } @@ -39,11 +24,8 @@ module "karpenter" { cluster_name = module.eks.cluster_name - irsa_oidc_provider_arn = module.eks.oidc_provider_arn - irsa_namespace_service_accounts = ["karpenter:karpenter"] - # Attach additional IAM policies to the Karpenter node IAM role - iam_role_additional_policies = { + node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } @@ -54,15 +36,13 @@ module "karpenter" { } ``` -### External Node IAM Role (Default) +### Re-Use Existing Node IAM Role In the following example, the Karpenter module will create: -- An IAM role for service accounts (IRSA) with a narrowly scoped IAM policy for the Karpenter controller to utilize -- An IAM instance profile for the nodes created by Karpenter to utilize - - Note: This setup will utilize the existing IAM role created by the EKS Managed Node group which means the role is already populated in the `aws-auth` configmap and no further updates are required. -- An SQS queue and Eventbridge event rules for Karpenter to utilize for spot termination handling, capacity rebalancing, etc. +- An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller +- SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. -In this scenario, Karpenter would run atop the EKS Managed Node group and scale out nodes as needed from there: +In this scenario, Karpenter will re-use an existing Node IAM role from the EKS managed nodegroup which already has the necessary access entry permissions: ```hcl module "eks" { @@ -86,12 +66,12 @@ module "karpenter" { cluster_name = module.eks.cluster_name - irsa_oidc_provider_arn = module.eks.oidc_provider_arn - irsa_namespace_service_accounts = ["karpenter:karpenter"] - create_iam_role = false iam_role_arn = module.eks.eks_managed_node_groups["initial"].iam_role_arn + # Since the nodegroup role will already have an access entry + create_access_entry = false + tags = { Environment = "dev" Terraform = "true" @@ -104,14 +84,14 @@ module "karpenter" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.34 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.57 | +| [aws](#provider\_aws) | >= 5.34 | ## Modules @@ -123,61 +103,65 @@ No modules. |------|------| | [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_eks_access_entry.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | | [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | -| [aws_iam_policy.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.irsa_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_policy.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.controller_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.node_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_sqs_queue.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | | [aws_sqs_queue_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.irsa_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.controller_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.node_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | +| [access\_entry\_type](#input\_access\_entry\_type) | Type of the access entry. `EC2_LINUX`, `FARGATE_LINUX`, or `EC2_WINDOWS`; defaults to `EC2_LINUX` | `string` | `"EC2_LINUX"` | no | +| [ami\_id\_ssm\_parameter\_arns](#input\_ami\_id\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that Karpenter controller is allowed read access (for retrieving AMI IDs) | `list(string)` | `[]` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. Note: If `ipv6` is specified, the `AmazonEKS_CNI_IPv6_Policy` must exist in the account. This policy is created by the EKS module with `create_cni_ipv6_iam_policy = true` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | The name of the EKS cluster | `string` | `""` | no | -| [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | -| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | -| [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `true` | no | -| [create\_irsa](#input\_create\_irsa) | Determines whether an IAM role for service accounts is created | `bool` | `true` | no | -| [enable\_karpenter\_instance\_profile\_creation](#input\_enable\_karpenter\_instance\_profile\_creation) | Determines whether Karpenter will be allowed to create the IAM instance profile (v1beta1) or if Terraform will (v1alpha1) | `bool` | `false` | no | +| [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | +| [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the node IAM role | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created | `bool` | `true` | no | +| [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `false` | no | +| [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [enable\_irsa](#input\_enable\_irsa) | Determines whether to enable support IAM role for service account | `bool` | `false` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | -| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | -| [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | -| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | -| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_policy\_description](#input\_iam\_policy\_description) | IAM policy description | `string` | `"Karpenter controller IAM policy"` | no | +| [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy | `string` | `"KarpenterController"` | no | +| [iam\_policy\_path](#input\_iam\_policy\_path) | Path of the IAM policy | `string` | `"/"` | no | +| [iam\_policy\_use\_name\_prefix](#input\_iam\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`iam_policy_name`) is used as a prefix | `bool` | `true` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | IAM role description | `string` | `"Karpenter controller IAM role"` | no | | [iam\_role\_max\_session\_duration](#input\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | -| [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | -| [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `"/"` | no | -| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | -| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | -| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name of the IAM role | `string` | `"KarpenterController"` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | Path of the IAM role | `string` | `"/"` | no | +| [iam\_role\_permissions\_boundary\_arn](#input\_iam\_role\_permissions\_boundary\_arn) | Permissions boundary ARN to use for the IAM role | `string` | `null` | no | +| [iam\_role\_policies](#input\_iam\_role\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add the the IAM role | `map(any)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the name of the IAM role (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no | -| [irsa\_description](#input\_irsa\_description) | IAM role for service accounts description | `string` | `"Karpenter IAM role for service account"` | no | -| [irsa\_max\_session\_duration](#input\_irsa\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | -| [irsa\_name](#input\_irsa\_name) | Name of IAM role for service accounts | `string` | `null` | no | | [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` |
[
"karpenter:karpenter"
]
| no | | [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | -| [irsa\_path](#input\_irsa\_path) | Path of IAM role for service accounts | `string` | `"/"` | no | -| [irsa\_permissions\_boundary\_arn](#input\_irsa\_permissions\_boundary\_arn) | Permissions boundary ARN to use for IAM role for service accounts | `string` | `null` | no | -| [irsa\_policy\_name](#input\_irsa\_policy\_name) | Name of IAM policy for service accounts | `string` | `null` | no | -| [irsa\_ssm\_parameter\_arns](#input\_irsa\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter | `list(string)` |
[
"arn:aws:ssm:*:*:parameter/aws/service/*"
]
| no | -| [irsa\_subnet\_account\_id](#input\_irsa\_subnet\_account\_id) | Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account | `string` | `""` | no | -| [irsa\_tag\_key](#input\_irsa\_tag\_key) | Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner | `string` | `"karpenter.sh/discovery"` | no | -| [irsa\_tag\_values](#input\_irsa\_tag\_values) | Tag values (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner. Defaults to cluster name when not set. | `list(string)` | `[]` | no | -| [irsa\_tags](#input\_irsa\_tags) | A map of additional tags to add the the IAM role for service accounts | `map(any)` | `{}` | no | -| [irsa\_use\_name\_prefix](#input\_irsa\_use\_name\_prefix) | Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix | `bool` | `true` | no | -| [policies](#input\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | +| [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | +| [node\_iam\_role\_arn](#input\_node\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [node\_iam\_role\_attach\_cni\_policy](#input\_node\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | +| [node\_iam\_role\_description](#input\_node\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [node\_iam\_role\_max\_session\_duration](#input\_node\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | +| [node\_iam\_role\_name](#input\_node\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | +| [node\_iam\_role\_path](#input\_node\_iam\_role\_path) | IAM role path | `string` | `"/"` | no | +| [node\_iam\_role\_permissions\_boundary](#input\_node\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [node\_iam\_role\_tags](#input\_node\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [node\_iam\_role\_use\_name\_prefix](#input\_node\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [queue\_kms\_data\_key\_reuse\_period\_seconds](#input\_queue\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again | `number` | `null` | no | | [queue\_kms\_master\_key\_id](#input\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no | | [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | @@ -190,17 +174,18 @@ No modules. | Name | Description | |------|-------------| | [event\_rules](#output\_event\_rules) | Map of the event rules created and their attributes | +| [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the controller IAM role | +| [iam\_role\_name](#output\_iam\_role\_name) | The name of the controller IAM role | +| [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the controller IAM role | | [instance\_profile\_arn](#output\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | | [instance\_profile\_id](#output\_instance\_profile\_id) | Instance profile's ID | | [instance\_profile\_name](#output\_instance\_profile\_name) | Name of the instance profile | | [instance\_profile\_unique](#output\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | -| [irsa\_arn](#output\_irsa\_arn) | The Amazon Resource Name (ARN) specifying the IAM role for service accounts | -| [irsa\_name](#output\_irsa\_name) | The name of the IAM role for service accounts | -| [irsa\_unique\_id](#output\_irsa\_unique\_id) | Stable and unique string identifying the IAM role for service accounts | +| [node\_access\_entry\_arn](#output\_node\_access\_entry\_arn) | Amazon Resource Name (ARN) of the node Access Entry | +| [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | +| [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | The name of the node IAM role | +| [node\_iam\_role\_unique\_id](#output\_node\_iam\_role\_unique\_id) | Stable and unique string identifying the node IAM role | | [queue\_arn](#output\_queue\_arn) | The ARN of the SQS queue | | [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue | | [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue | -| [role\_arn](#output\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | -| [role\_name](#output\_role\_name) | The name of the IAM role | -| [role\_unique\_id](#output\_role\_unique\_id) | Stable and unique string identifying the IAM role | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 2ce9f95c16..dfd6042e34 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -1,207 +1,406 @@ +data "aws_region" "current" {} data "aws_partition" "current" {} data "aws_caller_identity" "current" {} locals { account_id = data.aws_caller_identity.current.account_id - partition = data.aws_partition.current.partition dns_suffix = data.aws_partition.current.dns_suffix + partition = data.aws_partition.current.partition + region = data.aws_region.current.name } ################################################################################ -# IAM Role for Service Account (IRSA) -# This is used by the Karpenter controller +# Karpenter controller IAM Role ################################################################################ locals { - create_irsa = var.create && var.create_irsa - irsa_name = coalesce(var.irsa_name, "KarpenterIRSA-${var.cluster_name}") - irsa_policy_name = coalesce(var.irsa_policy_name, local.irsa_name) - + create_iam_role = var.create && var.create_iam_role irsa_oidc_provider_url = replace(var.irsa_oidc_provider_arn, "/^(.*provider/)/", "") } -data "aws_iam_policy_document" "irsa_assume_role" { - count = local.create_irsa ? 1 : 0 +data "aws_iam_policy_document" "controller_assume_role" { + count = local.create_iam_role ? 1 : 0 + # Pod Identity statement { - effect = "Allow" - actions = ["sts:AssumeRoleWithWebIdentity"] + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] principals { - type = "Federated" - identifiers = [var.irsa_oidc_provider_arn] + type = "Service" + identifiers = ["pods.eks.amazonaws.com"] } + } - condition { - test = var.irsa_assume_role_condition_test - variable = "${local.irsa_oidc_provider_url}:sub" - values = [for sa in var.irsa_namespace_service_accounts : "system:serviceaccount:${sa}"] - } + # IAM Roles for Service Accounts (IRSA) + dynamic "statement" { + for_each = var.enable_irsa ? [1] : [] - # https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-oidc-and-irsa/?nc1=h_ls - condition { - test = var.irsa_assume_role_condition_test - variable = "${local.irsa_oidc_provider_url}:aud" - values = ["sts.amazonaws.com"] + content { + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = [var.irsa_oidc_provider_arn] + } + + condition { + test = var.irsa_assume_role_condition_test + variable = "${local.irsa_oidc_provider_url}:sub" + values = [for sa in var.irsa_namespace_service_accounts : "system:serviceaccount:${sa}"] + } + + # https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-oidc-and-irsa/?nc1=h_ls + condition { + test = var.irsa_assume_role_condition_test + variable = "${local.irsa_oidc_provider_url}:aud" + values = ["sts.amazonaws.com"] + } } } } -resource "aws_iam_role" "irsa" { - count = local.create_irsa ? 1 : 0 +resource "aws_iam_role" "controller" { + count = local.create_iam_role ? 1 : 0 - name = var.irsa_use_name_prefix ? null : local.irsa_name - name_prefix = var.irsa_use_name_prefix ? "${local.irsa_name}-" : null - path = var.irsa_path - description = var.irsa_description + name = var.iam_role_use_name_prefix ? null : var.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${var.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description - assume_role_policy = data.aws_iam_policy_document.irsa_assume_role[0].json - max_session_duration = var.irsa_max_session_duration - permissions_boundary = var.irsa_permissions_boundary_arn + assume_role_policy = data.aws_iam_policy_document.controller_assume_role[0].json + max_session_duration = var.iam_role_max_session_duration + permissions_boundary = var.iam_role_permissions_boundary_arn force_detach_policies = true - tags = merge(var.tags, var.irsa_tags) + tags = merge(var.tags, var.iam_role_tags) } -locals { - irsa_tag_values = coalescelist(var.irsa_tag_values, [var.cluster_name]) -} +data "aws_iam_policy_document" "controller" { + count = local.create_iam_role ? 1 : 0 -data "aws_iam_policy_document" "irsa" { - count = local.create_irsa ? 1 : 0 + statement { + sid = "AllowScopedEC2InstanceActions" + resources = [ + "arn:${local.partition}:ec2:*::image/*", + "arn:${local.partition}:ec2:*::snapshot/*", + "arn:${local.partition}:ec2:*:*:spot-instances-request/*", + "arn:${local.partition}:ec2:*:*:security-group/*", + "arn:${local.partition}:ec2:*:*:subnet/*", + "arn:${local.partition}:ec2:*:*:launch-template/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + } statement { + sid = "AllowScopedEC2InstanceActionsWithTags" + resources = [ + "arn:${local.partition}:ec2:*:*:fleet/*", + "arn:${local.partition}:ec2:*:*:instance/*", + "arn:${local.partition}:ec2:*:*:volume/*", + "arn:${local.partition}:ec2:*:*:network-interface/*", + "arn:${local.partition}:ec2:*:*:launch-template/*", + "arn:${local.partition}:ec2:*:*:spot-instances-request/*", + ] actions = [ - "ec2:CreateLaunchTemplate", + "ec2:RunInstances", "ec2:CreateFleet", - "ec2:CreateTags", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeImages", - "ec2:DescribeInstances", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeInstanceTypes", - "ec2:DescribeInstanceTypeOfferings", - "ec2:DescribeAvailabilityZones", - "ec2:DescribeSpotPriceHistory", - "pricing:GetProducts", + "ec2:CreateLaunchTemplate" ] - resources = ["*"] + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } } statement { - actions = [ - "ec2:TerminateInstances", - "ec2:DeleteLaunchTemplate", + sid = "AllowScopedResourceCreationTagging" + resources = [ + "arn:${local.partition}:ec2:*:*:fleet/*", + "arn:${local.partition}:ec2:*:*:instance/*", + "arn:${local.partition}:ec2:*:*:volume/*", + "arn:${local.partition}:ec2:*:*:network-interface/*", + "arn:${local.partition}:ec2:*:*:launch-template/*", + "arn:${local.partition}:ec2:*:*:spot-instances-request/*", ] + actions = ["ec2:CreateTags"] - resources = ["*"] + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } condition { test = "StringEquals" - variable = "ec2:ResourceTag/${var.irsa_tag_key}" - values = local.irsa_tag_values + variable = "ec2:CreateAction" + values = [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate", + ] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] } } statement { - actions = ["ec2:RunInstances"] + sid = "AllowScopedResourceTagging" + resources = ["arn:${local.partition}:ec2:*:*:instance/*"] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = [ + "karpenter.sh/nodeclaim", + "Name", + ] + } + } + + statement { + sid = "AllowScopedDeletion" resources = [ - "arn:${local.partition}:ec2:*:${local.account_id}:launch-template/*", + "arn:${local.partition}:ec2:*:*:instance/*", + "arn:${local.partition}:ec2:*:*:launch-template/*" + ] + + actions = [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" ] condition { test = "StringEquals" - variable = "ec2:ResourceTag/${var.irsa_tag_key}" - values = local.irsa_tag_values + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] } } statement { - actions = ["ec2:RunInstances"] - resources = [ - "arn:${local.partition}:ec2:*::image/*", - "arn:${local.partition}:ec2:*::snapshot/*", - "arn:${local.partition}:ec2:*:${local.account_id}:instance/*", - "arn:${local.partition}:ec2:*:${local.account_id}:spot-instances-request/*", - "arn:${local.partition}:ec2:*:${local.account_id}:security-group/*", - "arn:${local.partition}:ec2:*:${local.account_id}:volume/*", - "arn:${local.partition}:ec2:*:${local.account_id}:network-interface/*", - "arn:${local.partition}:ec2:*:${coalesce(var.irsa_subnet_account_id, local.account_id)}:subnet/*", + sid = "AllowRegionalReadActions" + resources = ["*"] + actions = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" ] + + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [local.region] + } } statement { + sid = "AllowSSMReadActions" + resources = coalescelist(var.ami_id_ssm_parameter_arns, ["arn:${local.partition}:ssm:${local.region}::parameter/aws/service/*"]) actions = ["ssm:GetParameter"] - resources = var.irsa_ssm_parameter_arns } statement { - actions = ["eks:DescribeCluster"] - resources = ["arn:${local.partition}:eks:*:${local.account_id}:cluster/${var.cluster_name}"] + sid = "AllowPricingReadActions" + resources = ["*"] + actions = ["pricing:GetProducts"] } statement { + sid = "AllowInterruptionQueueActions" + resources = [aws_sqs_queue.this[0].arn] + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] + } + + statement { + sid = "AllowPassingInstanceRole" + resources = var.create_node_iam_role ? [aws_iam_role.node[0].arn] : [var.node_iam_role_arn] actions = ["iam:PassRole"] - resources = [var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["ec2.amazonaws.com"] + } } - dynamic "statement" { - for_each = local.enable_spot_termination ? [1] : [] + statement { + sid = "AllowScopedInstanceProfileCreationActions" + resources = ["*"] + actions = ["iam:CreateInstanceProfile"] - content { - actions = [ - "sqs:DeleteMessage", - "sqs:GetQueueUrl", - "sqs:GetQueueAttributes", - "sqs:ReceiveMessage", - ] - resources = [aws_sqs_queue.this[0].arn] + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] } } - # TODO - this will be replaced in v20.0 with the scoped policy provided by Karpenter - # https://github.com/aws/karpenter/blob/main/website/content/en/docs/upgrading/v1beta1-controller-policy.json - dynamic "statement" { - for_each = var.enable_karpenter_instance_profile_creation ? [1] : [] + statement { + sid = "AllowScopedInstanceProfileTagActions" + resources = ["*"] + actions = ["iam:TagInstanceProfile"] - content { - actions = [ - "iam:AddRoleToInstanceProfile", - "iam:CreateInstanceProfile", - "iam:DeleteInstanceProfile", - "iam:GetInstanceProfile", - "iam:RemoveRoleFromInstanceProfile", - "iam:TagInstanceProfile", - ] - resources = ["*"] + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] } } + + statement { + sid = "AllowScopedInstanceProfileActions" + resources = ["*"] + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowInstanceProfileReadActions" + resources = ["*"] + actions = ["iam:GetInstanceProfile"] + } + + statement { + sid = "AllowAPIServerEndpointDiscovery" + resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] + actions = ["eks:DescribeCluster"] + } } -resource "aws_iam_policy" "irsa" { - count = local.create_irsa ? 1 : 0 +resource "aws_iam_policy" "controller" { + count = local.create_iam_role ? 1 : 0 - name_prefix = "${local.irsa_policy_name}-" - path = var.irsa_path - description = var.irsa_description - policy = data.aws_iam_policy_document.irsa[0].json + name = var.iam_policy_use_name_prefix ? null : var.iam_policy_name + name_prefix = var.iam_policy_use_name_prefix ? "${var.iam_policy_name}-" : null + path = var.iam_policy_path + description = var.iam_policy_description + policy = data.aws_iam_policy_document.controller[0].json tags = var.tags } -resource "aws_iam_role_policy_attachment" "irsa" { - count = local.create_irsa ? 1 : 0 +resource "aws_iam_role_policy_attachment" "controller" { + count = local.create_iam_role ? 1 : 0 - role = aws_iam_role.irsa[0].name - policy_arn = aws_iam_policy.irsa[0].arn + role = aws_iam_role.controller[0].name + policy_arn = aws_iam_policy.controller[0].arn } -resource "aws_iam_role_policy_attachment" "irsa_additional" { - for_each = { for k, v in var.policies : k => v if local.create_irsa } +resource "aws_iam_role_policy_attachment" "controller_additional" { + for_each = { for k, v in var.iam_role_policies : k => v if local.create_iam_role } - role = aws_iam_role.irsa[0].name + role = aws_iam_role.controller[0].name policy_arn = each.value } @@ -266,7 +465,7 @@ locals { detail-type = ["AWS Health Event"] } } - spot_interupt = { + spot_interrupt = { name = "SpotInterrupt" description = "Karpenter interrupt - EC2 spot instance interruption warning" event_pattern = { @@ -320,15 +519,15 @@ resource "aws_cloudwatch_event_target" "this" { ################################################################################ locals { - create_iam_role = var.create && var.create_iam_role + create_node_iam_role = var.create && var.create_node_iam_role - iam_role_name = coalesce(var.iam_role_name, "Karpenter-${var.cluster_name}") - iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" - cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${local.partition}:iam::${local.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + node_iam_role_name = coalesce(var.node_iam_role_name, "Karpenter-${var.cluster_name}") + node_iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" + cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${local.partition}:iam::${local.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.node_iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } -data "aws_iam_policy_document" "assume_role" { - count = local.create_iam_role ? 1 : 0 +data "aws_iam_policy_document" "node_assume_role" { + count = local.create_node_iam_role ? 1 : 0 statement { sid = "EKSNodeAssumeRole" @@ -336,62 +535,83 @@ data "aws_iam_policy_document" "assume_role" { principals { type = "Service" - identifiers = ["ec2.${local.dns_suffix}"] + identifiers = ["ec2.amazonaws.com"] } } } -resource "aws_iam_role" "this" { - count = local.create_iam_role ? 1 : 0 +resource "aws_iam_role" "node" { + count = local.create_node_iam_role ? 1 : 0 - name = var.iam_role_use_name_prefix ? null : local.iam_role_name - name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null - path = var.iam_role_path - description = var.iam_role_description + name = var.node_iam_role_use_name_prefix ? null : local.node_iam_role_name + name_prefix = var.node_iam_role_use_name_prefix ? "${local.node_iam_role_name}-" : null + path = var.node_iam_role_path + description = var.node_iam_role_description - assume_role_policy = data.aws_iam_policy_document.assume_role[0].json - max_session_duration = var.iam_role_max_session_duration - permissions_boundary = var.iam_role_permissions_boundary + assume_role_policy = data.aws_iam_policy_document.node_assume_role[0].json + max_session_duration = var.node_iam_role_max_session_duration + permissions_boundary = var.node_iam_role_permissions_boundary force_detach_policies = true - tags = merge(var.tags, var.iam_role_tags) + tags = merge(var.tags, var.node_iam_role_tags) } # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group -resource "aws_iam_role_policy_attachment" "this" { +resource "aws_iam_role_policy_attachment" "node" { for_each = { for k, v in toset(compact([ - "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", - "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", - var.iam_role_attach_cni_policy ? local.cni_policy : "", - ])) : k => v if local.create_iam_role } + "${local.node_iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", + "${local.node_iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", + var.node_iam_role_attach_cni_policy ? local.cni_policy : "", + ])) : k => v if local.create_node_iam_role } policy_arn = each.value - role = aws_iam_role.this[0].name + role = aws_iam_role.node[0].name } -resource "aws_iam_role_policy_attachment" "additional" { - for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_role } +resource "aws_iam_role_policy_attachment" "node_additional" { + for_each = { for k, v in var.node_iam_role_additional_policies : k => v if local.create_node_iam_role } policy_arn = each.value - role = aws_iam_role.this[0].name + role = aws_iam_role.node[0].name +} + +################################################################################ +# Access Entry +################################################################################ + +resource "aws_eks_access_entry" "node" { + count = var.create && var.create_access_entry ? 1 : 0 + + cluster_name = var.cluster_name + principal_arn = var.create_node_iam_role ? aws_iam_role.node[0].arn : var.node_iam_role_arn + type = var.access_entry_type + + tags = var.tags + + depends_on = [ + # If we try to add this too quickly, it fails. So .... we wait + aws_sqs_queue_policy.this, + ] } ################################################################################ # Node IAM Instance Profile # This is used by the nodes launched by Karpenter +# Starting with Karpenter 0.32 this is no longer required as Karpenter will +# create the Instance Profile ################################################################################ locals { - external_role_name = try(replace(var.iam_role_arn, "/^(.*role/)/", ""), null) + external_role_name = try(replace(var.node_iam_role_arn, "/^(.*role/)/", ""), null) } resource "aws_iam_instance_profile" "this" { count = var.create && var.create_instance_profile ? 1 : 0 - name = var.iam_role_use_name_prefix ? null : local.iam_role_name - name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null - path = var.iam_role_path - role = var.create_iam_role ? aws_iam_role.this[0].name : local.external_role_name + name = var.node_iam_role_use_name_prefix ? null : local.node_iam_role_name + name_prefix = var.node_iam_role_use_name_prefix ? "${local.node_iam_role_name}-" : null + path = var.node_iam_role_path + role = var.create_node_iam_role ? aws_iam_role.node[0].name : local.external_role_name - tags = merge(var.tags, var.iam_role_tags) + tags = merge(var.tags, var.node_iam_role_tags) } diff --git a/modules/karpenter/migrations.tf b/modules/karpenter/migrations.tf new file mode 100644 index 0000000000..3e7ca59a9b --- /dev/null +++ b/modules/karpenter/migrations.tf @@ -0,0 +1,56 @@ +################################################################################ +# Migrations: v19.21 -> v20.0 +################################################################################ + +# Node IAM role +moved { + from = aws_iam_role.this + to = aws_iam_role.node +} + +moved { + from = aws_iam_policy.this + to = aws_iam_policy.node +} + +moved { + from = aws_iam_role_policy_attachment.this + to = aws_iam_role_policy_attachment.node +} + +moved { + from = aws_iam_role_policy_attachment.additional + to = aws_iam_role_policy_attachment.node_additional +} + +# Controller IAM role +moved { + from = aws_iam_role.irsa + to = aws_iam_role.controller +} + +moved { + from = aws_iam_policy.irsa + to = aws_iam_policy.controller +} + +moved { + from = aws_iam_role_policy_attachment.irsa + to = aws_iam_role_policy_attachment.controller +} + +moved { + from = aws_iam_role_policy_attachment.irsa_additional + to = aws_iam_role_policy_attachment.controller_additional +} + +# Spelling correction +moved { + from = aws_cloudwatch_event_target.this["spot_interupt"] + to = aws_cloudwatch_event_target.this["spot_interrupt"] +} + +moved { + from = aws_cloudwatch_event_rule.this["spot_interupt"] + to = aws_cloudwatch_event_rule.this["spot_interrupt"] +} diff --git a/modules/karpenter/outputs.tf b/modules/karpenter/outputs.tf index 947de39bfd..164baa142c 100644 --- a/modules/karpenter/outputs.tf +++ b/modules/karpenter/outputs.tf @@ -1,20 +1,20 @@ ################################################################################ -# IAM Role for Service Account (IRSA) +# Karpenter controller IAM Role ################################################################################ -output "irsa_name" { - description = "The name of the IAM role for service accounts" - value = try(aws_iam_role.irsa[0].name, null) +output "iam_role_name" { + description = "The name of the controller IAM role" + value = try(aws_iam_role.controller[0].name, null) } -output "irsa_arn" { - description = "The Amazon Resource Name (ARN) specifying the IAM role for service accounts" - value = try(aws_iam_role.irsa[0].arn, null) +output "iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the controller IAM role" + value = try(aws_iam_role.controller[0].arn, null) } -output "irsa_unique_id" { - description = "Stable and unique string identifying the IAM role for service accounts" - value = try(aws_iam_role.irsa[0].unique_id, null) +output "iam_role_unique_id" { + description = "Stable and unique string identifying the controller IAM role" + value = try(aws_iam_role.controller[0].unique_id, null) } ################################################################################ @@ -49,19 +49,28 @@ output "event_rules" { # Node IAM Role ################################################################################ -output "role_name" { - description = "The name of the IAM role" - value = try(aws_iam_role.this[0].name, null) +output "node_iam_role_name" { + description = "The name of the node IAM role" + value = try(aws_iam_role.node[0].name, null) } -output "role_arn" { - description = "The Amazon Resource Name (ARN) specifying the IAM role" - value = try(aws_iam_role.this[0].arn, var.iam_role_arn) +output "node_iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the node IAM role" + value = try(aws_iam_role.node[0].arn, var.node_iam_role_arn) } -output "role_unique_id" { - description = "Stable and unique string identifying the IAM role" - value = try(aws_iam_role.this[0].unique_id, null) +output "node_iam_role_unique_id" { + description = "Stable and unique string identifying the node IAM role" + value = try(aws_iam_role.node[0].unique_id, null) +} + +################################################################################ +# Access Entry +################################################################################ + +output "node_access_entry_arn" { + description = "Amazon Resource Name (ARN) of the node Access Entry" + value = try(aws_eks_access_entry.node[0].access_entry_arn, null) } ################################################################################ diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 4a8389671b..3af82d4fc6 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -1,5 +1,5 @@ variable "create" { - description = "Determines whether to create EKS managed node group or not" + description = "Controls if resources should be created (affects nearly all resources)" type = bool default = true } @@ -17,92 +17,101 @@ variable "cluster_name" { } ################################################################################ -# IAM Role for Service Account (IRSA) +# Karpenter controller IAM Role ################################################################################ -variable "create_irsa" { - description = "Determines whether an IAM role for service accounts is created" +variable "create_iam_role" { + description = "Determines whether an IAM role is created" type = bool default = true } -variable "irsa_name" { - description = "Name of IAM role for service accounts" - type = string - default = null -} - -variable "irsa_policy_name" { - description = "Name of IAM policy for service accounts" +variable "iam_role_name" { + description = "Name of the IAM role" type = string - default = null + default = "KarpenterController" } -variable "irsa_use_name_prefix" { - description = "Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix" +variable "iam_role_use_name_prefix" { + description = "Determines whether the name of the IAM role (`iam_role_name`) is used as a prefix" type = bool default = true } -variable "irsa_path" { - description = "Path of IAM role for service accounts" +variable "iam_role_path" { + description = "Path of the IAM role" type = string default = "/" } -variable "irsa_description" { - description = "IAM role for service accounts description" +variable "iam_role_description" { + description = "IAM role description" type = string - default = "Karpenter IAM role for service account" + default = "Karpenter controller IAM role" } -variable "irsa_max_session_duration" { +variable "iam_role_max_session_duration" { description = "Maximum API session duration in seconds between 3600 and 43200" type = number default = null } -variable "irsa_permissions_boundary_arn" { - description = "Permissions boundary ARN to use for IAM role for service accounts" +variable "iam_role_permissions_boundary_arn" { + description = "Permissions boundary ARN to use for the IAM role" type = string default = null } -variable "irsa_tags" { - description = "A map of additional tags to add the the IAM role for service accounts" +variable "iam_role_tags" { + description = "A map of additional tags to add the the IAM role" type = map(any) default = {} } -variable "policies" { - description = "Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format" - type = map(string) - default = {} +variable "iam_policy_name" { + description = "Name of the IAM policy" + type = string + default = "KarpenterController" +} + +variable "iam_policy_use_name_prefix" { + description = "Determines whether the name of the IAM policy (`iam_policy_name`) is used as a prefix" + type = bool + default = true } -variable "irsa_tag_key" { - description = "Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner" +variable "iam_policy_path" { + description = "Path of the IAM policy" type = string - default = "karpenter.sh/discovery" + default = "/" } -variable "irsa_tag_values" { - description = "Tag values (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner. Defaults to cluster name when not set." - type = list(string) - default = [] +variable "iam_policy_description" { + description = "IAM policy description" + type = string + default = "Karpenter controller IAM policy" } -variable "irsa_ssm_parameter_arns" { - description = "List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter" +variable "iam_role_policies" { + description = "Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format" + type = map(string) + default = {} +} + +variable "ami_id_ssm_parameter_arns" { + description = "List of SSM Parameter ARNs that Karpenter controller is allowed read access (for retrieving AMI IDs)" type = list(string) - # https://github.com/aws/karpenter/blob/ed9473a9863ca949b61b9846c8b9f33f35b86dbd/pkg/cloudprovider/aws/ami.go#L105-L123 - default = ["arn:aws:ssm:*:*:parameter/aws/service/*"] + default = [] } -variable "irsa_subnet_account_id" { - description = "Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account" - type = string - default = "" +################################################################################ +# IAM Role for Service Account (IRSA) +################################################################################ + +variable "enable_irsa" { + description = "Determines whether to enable support IAM role for service account" + type = bool + default = false } variable "irsa_oidc_provider_arn" { @@ -123,12 +132,6 @@ variable "irsa_assume_role_condition_test" { default = "StringEquals" } -variable "enable_karpenter_instance_profile_creation" { - description = "Determines whether Karpenter will be allowed to create the IAM instance profile (v1beta1) or if Terraform will (v1alpha1)" - type = bool - default = false -} - ################################################################################ # Node Termination Queue ################################################################################ @@ -164,81 +167,97 @@ variable "queue_kms_data_key_reuse_period_seconds" { } ################################################################################ -# Node IAM Role & Instance Profile +# Node IAM Role ################################################################################ -variable "create_iam_role" { +variable "create_node_iam_role" { description = "Determines whether an IAM role is created or to use an existing IAM role" type = bool default = true } variable "cluster_ip_family" { - description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. Note: If `ipv6` is specified, the `AmazonEKS_CNI_IPv6_Policy` must exist in the account. This policy is created by the EKS module with `create_cni_ipv6_iam_policy = true`" type = string default = null } -variable "iam_role_arn" { +variable "node_iam_role_arn" { description = "Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false`" type = string default = null } -variable "iam_role_name" { +variable "node_iam_role_name" { description = "Name to use on IAM role created" type = string default = null } -variable "iam_role_use_name_prefix" { +variable "node_iam_role_use_name_prefix" { description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" type = bool default = true } -variable "iam_role_path" { +variable "node_iam_role_path" { description = "IAM role path" type = string default = "/" } -variable "iam_role_description" { +variable "node_iam_role_description" { description = "Description of the role" type = string default = null } -variable "iam_role_max_session_duration" { +variable "node_iam_role_max_session_duration" { description = "Maximum API session duration in seconds between 3600 and 43200" type = number default = null } -variable "iam_role_permissions_boundary" { +variable "node_iam_role_permissions_boundary" { description = "ARN of the policy that is used to set the permissions boundary for the IAM role" type = string default = null } -variable "iam_role_attach_cni_policy" { +variable "node_iam_role_attach_cni_policy" { description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" type = bool default = true } -variable "iam_role_additional_policies" { +variable "node_iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" type = map(string) default = {} } -variable "iam_role_tags" { +variable "node_iam_role_tags" { description = "A map of additional tags to add to the IAM role created" type = map(string) default = {} } +################################################################################ +# Access Entry +################################################################################ + +variable "create_access_entry" { + description = "Determines whether an access entry is created for the IAM role used by the node IAM role" + type = bool + default = true +} + +variable "access_entry_type" { + description = "Type of the access entry. `EC2_LINUX`, `FARGATE_LINUX`, or `EC2_WINDOWS`; defaults to `EC2_LINUX`" + type = string + default = "EC2_LINUX" +} + ################################################################################ # Node IAM Instance Profile ################################################################################ @@ -246,7 +265,7 @@ variable "iam_role_tags" { variable "create_instance_profile" { description = "Whether to create an IAM instance profile" type = bool - default = true + default = false } ################################################################################ diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 01d187af62..63e9319d92 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" + version = ">= 5.34" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 8964144994..83e44d0bc5 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -42,14 +42,14 @@ module "self_managed_node_group" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.57 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.34 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.57 | +| [aws](#provider\_aws) | >= 5.34 | ## Modules @@ -63,6 +63,7 @@ module "self_managed_node_group" { |------|------| | [aws_autoscaling_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | | [aws_autoscaling_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_schedule) | resource | +| [aws_eks_access_entry.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | | [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | @@ -93,6 +94,7 @@ module "self_managed_node_group" { | [context](#input\_context) | Reserved | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | | [create](#input\_create) | Determines whether to create self managed node group or not | `bool` | `true` | no | +| [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the nodegroup | `bool` | `true` | no | | [create\_autoscaling\_group](#input\_create\_autoscaling\_group) | Determines whether to create autoscaling group or not | `bool` | `true` | no | | [create\_iam\_instance\_profile](#input\_create\_iam\_instance\_profile) | Determines whether an IAM instance profile is created or to use an existing IAM instance profile | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create launch template or not | `bool` | `true` | no | @@ -116,6 +118,7 @@ module "self_managed_node_group" { | [hibernation\_options](#input\_hibernation\_options) | The hibernation options for the instance | `map(string)` | `{}` | no | | [iam\_instance\_profile\_arn](#input\_iam\_instance\_profile\_arn) | Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false` | `string` | `null` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | +| [iam\_role\_arn](#input\_iam\_role\_arn) | ARN of the IAM role used by the instance profile. Required when `create_access_entry = true` and `create_iam_instance_profile = false` | `string` | `null` | no | | [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | @@ -125,6 +128,7 @@ module "self_managed_node_group" { | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [initial\_lifecycle\_hooks](#input\_initial\_lifecycle\_hooks) | One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource | `list(map(string))` | `[]` | no | | [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | +| [instance\_maintenance\_policy](#input\_instance\_maintenance\_policy) | If this block is configured, add a instance maintenance policy to the specified Auto Scaling group | `any` | `{}` | no | | [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `{}` | no | | [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated | `any` |
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | | [instance\_requirements](#input\_instance\_requirements) | The attribute requirements for the type of instance. If present then `instance_type` cannot be present | `any` | `{}` | no | @@ -178,6 +182,7 @@ module "self_managed_node_group" { | Name | Description | |------|-------------| +| [access\_entry\_arn](#output\_access\_entry\_arn) | Amazon Resource Name (ARN) of the Access Entry | | [autoscaling\_group\_arn](#output\_autoscaling\_group\_arn) | The ARN for this autoscaling group | | [autoscaling\_group\_availability\_zones](#output\_autoscaling\_group\_availability\_zones) | The availability zones of the autoscaling group | | [autoscaling\_group\_default\_cooldown](#output\_autoscaling\_group\_default\_cooldown) | Time between a scaling activity and the succeeding scaling activity | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 3b04db8b80..eb3192970b 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -438,6 +438,15 @@ resource "aws_autoscaling_group" "this" { } } + dynamic "instance_maintenance_policy" { + for_each = length(var.instance_maintenance_policy) > 0 ? [var.instance_maintenance_policy] : [] + + content { + min_healthy_percentage = instance_maintenance_policy.value.min_healthy_percentage + max_healthy_percentage = instance_maintenance_policy.value.max_healthy_percentage + } + } + dynamic "instance_refresh" { for_each = length(var.instance_refresh) > 0 ? [var.instance_refresh] : [] @@ -446,11 +455,14 @@ resource "aws_autoscaling_group" "this" { for_each = try([instance_refresh.value.preferences], []) content { - checkpoint_delay = try(preferences.value.checkpoint_delay, null) - checkpoint_percentages = try(preferences.value.checkpoint_percentages, null) - instance_warmup = try(preferences.value.instance_warmup, null) - min_healthy_percentage = try(preferences.value.min_healthy_percentage, null) - skip_matching = try(preferences.value.skip_matching, null) + checkpoint_delay = try(preferences.value.checkpoint_delay, null) + checkpoint_percentages = try(preferences.value.checkpoint_percentages, null) + instance_warmup = try(preferences.value.instance_warmup, null) + max_healthy_percentage = try(preferences.value.max_healthy_percentage, null) + min_healthy_percentage = try(preferences.value.min_healthy_percentage, null) + scale_in_protected_instances = try(preferences.value.scale_in_protected_instances, null) + skip_matching = try(preferences.value.skip_matching, null) + standby_instances = try(preferences.value.standby_instances, null) } } @@ -686,28 +698,6 @@ resource "aws_autoscaling_group" "this" { } } -################################################################################ -# Autoscaling group schedule -################################################################################ - -resource "aws_autoscaling_schedule" "this" { - for_each = { for k, v in var.schedules : k => v if var.create && var.create_schedule } - - scheduled_action_name = each.key - autoscaling_group_name = aws_autoscaling_group.this[0].name - - min_size = try(each.value.min_size, null) - max_size = try(each.value.max_size, null) - desired_capacity = try(each.value.desired_size, null) - start_time = try(each.value.start_time, null) - end_time = try(each.value.end_time, null) - time_zone = try(each.value.time_zone, null) - - # [Minute] [Hour] [Day_of_Month] [Month_of_Year] [Day_of_Week] - # Cron examples: https://crontab.guru/examples.html - recurrence = try(each.value.recurrence, null) -} - ################################################################################ # IAM Role ################################################################################ @@ -727,7 +717,7 @@ data "aws_iam_policy_document" "assume_role_policy" { principals { type = "Service" - identifiers = ["ec2.${data.aws_partition.current.dns_suffix}"] + identifiers = ["ec2.amazonaws.com"] } } } @@ -780,3 +770,39 @@ resource "aws_iam_instance_profile" "this" { create_before_destroy = true } } + +################################################################################ +# Access Entry +################################################################################ + +resource "aws_eks_access_entry" "this" { + count = var.create && var.create_access_entry ? 1 : 0 + + cluster_name = var.cluster_name + principal_arn = var.create_iam_instance_profile ? aws_iam_role.this[0].arn : var.iam_role_arn + type = var.platform == "windows" ? "EC2_WINDOWS" : "EC2_LINUX" + + tags = var.tags +} + +################################################################################ +# Autoscaling group schedule +################################################################################ + +resource "aws_autoscaling_schedule" "this" { + for_each = { for k, v in var.schedules : k => v if var.create && var.create_schedule } + + scheduled_action_name = each.key + autoscaling_group_name = aws_autoscaling_group.this[0].name + + min_size = try(each.value.min_size, null) + max_size = try(each.value.max_size, null) + desired_capacity = try(each.value.desired_size, null) + start_time = try(each.value.start_time, null) + end_time = try(each.value.end_time, null) + time_zone = try(each.value.time_zone, null) + + # [Minute] [Hour] [Day_of_Month] [Month_of_Year] [Day_of_Week] + # Cron examples: https://crontab.guru/examples.html + recurrence = try(each.value.recurrence, null) +} diff --git a/modules/self-managed-node-group/outputs.tf b/modules/self-managed-node-group/outputs.tf index 5c83497218..aba5b13233 100644 --- a/modules/self-managed-node-group/outputs.tf +++ b/modules/self-managed-node-group/outputs.tf @@ -81,15 +81,6 @@ output "autoscaling_group_vpc_zone_identifier" { value = try(aws_autoscaling_group.this[0].vpc_zone_identifier, null) } -################################################################################ -# Autoscaling Group Schedule -################################################################################ - -output "autoscaling_group_schedule_arns" { - description = "ARNs of autoscaling group schedules" - value = { for k, v in aws_autoscaling_schedule.this : k => v.arn } -} - ################################################################################ # IAM Role ################################################################################ @@ -128,6 +119,24 @@ output "iam_instance_profile_unique" { value = try(aws_iam_instance_profile.this[0].unique_id, null) } +################################################################################ +# Access Entry +################################################################################ + +output "access_entry_arn" { + description = "Amazon Resource Name (ARN) of the Access Entry" + value = try(aws_eks_access_entry.this[0].access_entry_arn, null) +} + +################################################################################ +# Autoscaling Group Schedule +################################################################################ + +output "autoscaling_group_schedule_arns" { + description = "ARNs of autoscaling group schedules" + value = { for k, v in aws_autoscaling_schedule.this : k => v.arn } +} + ################################################################################ # Additional ################################################################################ diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 7e5d0cecb6..c6c5086a57 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -476,6 +476,12 @@ variable "initial_lifecycle_hooks" { default = [] } +variable "instance_maintenance_policy" { + description = "If this block is configured, add a instance maintenance policy to the specified Auto Scaling group" + type = any + default = {} +} + variable "instance_refresh" { description = "If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated" type = any @@ -517,22 +523,6 @@ variable "autoscaling_group_tags" { default = {} } -################################################################################ -# Autoscaling group schedule -################################################################################ - -variable "create_schedule" { - description = "Determines whether to create autoscaling group schedule or not" - type = bool - default = true -} - -variable "schedules" { - description = "Map of autoscaling group schedule to create" - type = map(any) - default = {} -} - ################################################################################ # IAM Role ################################################################################ @@ -602,3 +592,35 @@ variable "iam_role_tags" { type = map(string) default = {} } + +################################################################################ +# Access Entry +################################################################################ + +variable "create_access_entry" { + description = "Determines whether an access entry is created for the IAM role used by the nodegroup" + type = bool + default = true +} + +variable "iam_role_arn" { + description = "ARN of the IAM role used by the instance profile. Required when `create_access_entry = true` and `create_iam_instance_profile = false`" + type = string + default = null +} + +################################################################################ +# Autoscaling group schedule +################################################################################ + +variable "create_schedule" { + description = "Determines whether to create autoscaling group schedule or not" + type = bool + default = true +} + +variable "schedules" { + description = "Map of autoscaling group schedule to create" + type = map(any) + default = {} +} diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 01d187af62..63e9319d92 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" + version = ">= 5.34" } } } diff --git a/node_groups.tf b/node_groups.tf index db78861a81..ddfc20fa09 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -40,7 +40,6 @@ resource "time_sleep" "this" { ################################################################################ # EKS IPV6 CNI Policy -# TODO - hopefully AWS releases a managed policy which can replace this # https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy ################################################################################ @@ -62,7 +61,7 @@ data "aws_iam_policy_document" "cni_ipv6_policy" { statement { sid = "CreateTags" actions = ["ec2:CreateTags"] - resources = ["arn:${data.aws_partition.current.partition}:ec2:*:*:network-interface/*"] + resources = ["arn:${local.partition}:ec2:*:*:network-interface/*"] } } @@ -363,6 +362,7 @@ module "eks_managed_node_group" { # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.eks_managed_node_group_defaults, "iam_role_additional_policies", {})) + # Autoscaling group schedule create_schedule = try(each.value.create_schedule, var.eks_managed_node_group_defaults.create_schedule, true) schedules = try(each.value.schedules, var.eks_managed_node_group_defaults.schedules, {}) @@ -423,14 +423,12 @@ module "self_managed_node_group" { metrics_granularity = try(each.value.metrics_granularity, var.self_managed_node_group_defaults.metrics_granularity, null) service_linked_role_arn = try(each.value.service_linked_role_arn, var.self_managed_node_group_defaults.service_linked_role_arn, null) - initial_lifecycle_hooks = try(each.value.initial_lifecycle_hooks, var.self_managed_node_group_defaults.initial_lifecycle_hooks, []) - instance_refresh = try(each.value.instance_refresh, var.self_managed_node_group_defaults.instance_refresh, local.default_instance_refresh) - use_mixed_instances_policy = try(each.value.use_mixed_instances_policy, var.self_managed_node_group_defaults.use_mixed_instances_policy, false) - mixed_instances_policy = try(each.value.mixed_instances_policy, var.self_managed_node_group_defaults.mixed_instances_policy, null) - warm_pool = try(each.value.warm_pool, var.self_managed_node_group_defaults.warm_pool, {}) - - create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, true) - schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, {}) + initial_lifecycle_hooks = try(each.value.initial_lifecycle_hooks, var.self_managed_node_group_defaults.initial_lifecycle_hooks, []) + instance_maintenance_policy = try(each.value.instance_maintenance_policy, var.self_managed_node_group_defaults.instance_maintenance_policy, {}) + instance_refresh = try(each.value.instance_refresh, var.self_managed_node_group_defaults.instance_refresh, local.default_instance_refresh) + use_mixed_instances_policy = try(each.value.use_mixed_instances_policy, var.self_managed_node_group_defaults.use_mixed_instances_policy, false) + mixed_instances_policy = try(each.value.mixed_instances_policy, var.self_managed_node_group_defaults.mixed_instances_policy, null) + warm_pool = try(each.value.warm_pool, var.self_managed_node_group_defaults.warm_pool, {}) delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) autoscaling_group_tags = try(each.value.autoscaling_group_tags, var.self_managed_node_group_defaults.autoscaling_group_tags, {}) @@ -499,6 +497,14 @@ module "self_managed_node_group" { # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.self_managed_node_group_defaults, "iam_role_additional_policies", {})) + # Access entry + create_access_entry = try(each.value.create_access_entry, var.self_managed_node_group_defaults.create_access_entry, true) + iam_role_arn = try(each.value.iam_role_arn, var.self_managed_node_group_defaults.iam_role_arn, null) + + # Autoscaling group schedule + create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, true) + schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, {}) + # Security group vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.self_managed_node_group_defaults.vpc_security_group_ids, []))) cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, var.self_managed_node_group_defaults.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null diff --git a/outputs.tf b/outputs.tf index ea02a3a8cc..adba15c10c 100644 --- a/outputs.tf +++ b/outputs.tf @@ -52,6 +52,15 @@ output "cluster_primary_security_group_id" { value = try(aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id, null) } +################################################################################ +# Access Entry +################################################################################ + +output "access_entries" { + description = "Map of access entries created and their attributes" + value = aws_eks_access_entry.this +} + ################################################################################ # KMS Key ################################################################################ @@ -205,19 +214,3 @@ output "self_managed_node_groups_autoscaling_group_names" { description = "List of the autoscaling group names created by self-managed node groups" value = compact([for group in module.self_managed_node_group : group.autoscaling_group_name]) } - -################################################################################ -# Additional -################################################################################ - -output "aws_auth_configmap_yaml" { - description = "[DEPRECATED - use `var.manage_aws_auth_configmap`] Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" - value = templatefile("${path.module}/templates/aws_auth_cm.tpl", - { - eks_managed_role_arns = distinct(compact([for group in module.eks_managed_node_group : group.iam_role_arn])) - self_managed_role_arns = distinct(compact([for group in module.self_managed_node_group : group.iam_role_arn if group.platform != "windows"])) - win32_self_managed_role_arns = distinct(compact([for group in module.self_managed_node_group : group.iam_role_arn if group.platform == "windows"])) - fargate_profile_pod_execution_role_arns = distinct(compact([for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn])) - } - ) -} diff --git a/templates/aws_auth_cm.tpl b/templates/aws_auth_cm.tpl deleted file mode 100644 index 73a898e966..0000000000 --- a/templates/aws_auth_cm.tpl +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: aws-auth - namespace: kube-system -data: - mapRoles: | -%{ for role in eks_managed_role_arns ~} - - rolearn: ${role} - username: system:node:{{EC2PrivateDNSName}} - groups: - - system:bootstrappers - - system:nodes -%{ endfor ~} -%{ for role in self_managed_role_arns ~} - - rolearn: ${role} - username: system:node:{{EC2PrivateDNSName}} - groups: - - system:bootstrappers - - system:nodes -%{ endfor ~} -%{ for role in win32_self_managed_role_arns ~} - - rolearn: ${role} - username: system:node:{{EC2PrivateDNSName}} - groups: - - eks:kube-proxy-windows - - system:bootstrappers - - system:nodes -%{ endfor ~} -%{ for role in fargate_profile_pod_execution_role_arns ~} - - rolearn: ${role} - username: system:node:{{SessionName}} - groups: - - system:bootstrappers - - system:nodes - - system:node-proxier -%{ endfor ~} diff --git a/templates/windows_user_data.tpl b/templates/windows_user_data.tpl index 5000850604..9721d3cc33 100644 --- a/templates/windows_user_data.tpl +++ b/templates/windows_user_data.tpl @@ -1,5 +1,8 @@ +%{ if enable_bootstrap_user_data ~} +%{ endif ~} ${pre_bootstrap_user_data ~} +%{ if enable_bootstrap_user_data ~} [string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" [string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' [string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" @@ -7,3 +10,4 @@ ${pre_bootstrap_user_data ~} $LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } ${post_bootstrap_user_data ~} +%{ endif ~} diff --git a/variables.tf b/variables.tf index 810cee2bda..83776d6f4f 100644 --- a/variables.tf +++ b/variables.tf @@ -1,5 +1,5 @@ variable "create" { - description = "Controls if EKS resources should be created (affects nearly all resources)" + description = "Controls if resources should be created (affects nearly all resources)" type = bool default = true } @@ -38,6 +38,12 @@ variable "cluster_enabled_log_types" { default = ["audit", "api", "authenticator"] } +variable "authentication_mode" { + description = "The authentication mode for the cluster. Valid values are `CONFIG_MAP`, `API` or `API_AND_CONFIG_MAP`" + type = string + default = "API_AND_CONFIG_MAP" +} + variable "cluster_additional_security_group_ids" { description = "List of additional, externally created security group IDs to attach to the cluster control plane" type = list(string) @@ -130,6 +136,22 @@ variable "cluster_timeouts" { default = {} } +################################################################################ +# Access Entry +################################################################################ + +variable "access_entries" { + description = "Map of access entries to add to the cluster" + type = any + default = {} +} + +variable "enable_cluster_creator_admin_permissions" { + description = "Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry" + type = bool + default = false +} + ################################################################################ # KMS Key ################################################################################ @@ -153,15 +175,15 @@ variable "kms_key_deletion_window_in_days" { } variable "enable_kms_key_rotation" { - description = "Specifies whether key rotation is enabled. Defaults to `true`" + description = "Specifies whether key rotation is enabled" type = bool default = true } variable "kms_key_enable_default_policy" { - description = "Specifies whether to enable the default key policy. Defaults to `false`" + description = "Specifies whether to enable the default key policy" type = bool - default = false + default = true } variable "kms_key_owners" { @@ -228,6 +250,12 @@ variable "cloudwatch_log_group_kms_key_id" { default = null } +variable "cloudwatch_log_group_class" { + description = "Specified the log class of the log group. Possible values are: `STANDARD` or `INFREQUENT_ACCESS`" + type = string + default = null +} + variable "cloudwatch_log_group_tags" { description = "A map of additional tags to add to the cloudwatch log group created" type = map(string) @@ -428,14 +456,6 @@ variable "iam_role_additional_policies" { default = {} } -# TODO - hopefully this can be removed once the AWS endpoint is named properly in China -# https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1904 -variable "cluster_iam_role_dns_suffix" { - description = "Base DNS domain name for the current partition (e.g., amazonaws.com in AWS Commercial, amazonaws.com.cn in AWS China)" - type = string - default = null -} - variable "iam_role_tags" { description = "A map of additional tags to add to the IAM role created" type = map(string) @@ -557,55 +577,3 @@ variable "putin_khuylo" { type = bool default = true } - -################################################################################ -# aws-auth configmap -################################################################################ - -variable "manage_aws_auth_configmap" { - description = "Determines whether to manage the aws-auth configmap" - type = bool - default = false -} - -variable "create_aws_auth_configmap" { - description = "Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap`" - type = bool - default = false -} - -variable "aws_auth_node_iam_role_arns_non_windows" { - description = "List of non-Windows based node IAM role ARNs to add to the aws-auth configmap" - type = list(string) - default = [] -} - -variable "aws_auth_node_iam_role_arns_windows" { - description = "List of Windows based node IAM role ARNs to add to the aws-auth configmap" - type = list(string) - default = [] -} - -variable "aws_auth_fargate_profile_pod_execution_role_arns" { - description = "List of Fargate profile pod execution role ARNs to add to the aws-auth configmap" - type = list(string) - default = [] -} - -variable "aws_auth_roles" { - description = "List of role maps to add to the aws-auth configmap" - type = list(any) - default = [] -} - -variable "aws_auth_users" { - description = "List of user maps to add to the aws-auth configmap" - type = list(any) - default = [] -} - -variable "aws_auth_accounts" { - description = "List of account maps to add to the aws-auth configmap" - type = list(any) - default = [] -} diff --git a/versions.tf b/versions.tf index e3bdcbf6d7..a7084a7252 100644 --- a/versions.tf +++ b/versions.tf @@ -1,19 +1,15 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.3" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.57" + version = ">= 5.34" } tls = { source = "hashicorp/tls" version = ">= 3.0" } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" - } time = { source = "hashicorp/time" version = ">= 0.9" From a68aac6cf29011e95405207d409d03360d3af56d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 2 Feb 2024 14:36:53 +0000 Subject: [PATCH 723/962] chore(release): version 20.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [20.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.21.0...v20.0.0) (2024-02-02) ### ⚠ BREAKING CHANGES * Replace the use of `aws-auth` configmap with EKS cluster access entry (#2858) ### Features * Replace the use of `aws-auth` configmap with EKS cluster access entry ([#2858](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2858)) ([6b40bdb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6b40bdbb1d283d9259f43b03d24dca99cc1eceff)) --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcfa09ae70..2e7542610d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. +## [20.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.21.0...v20.0.0) (2024-02-02) + + +### ⚠ BREAKING CHANGES + +* Replace the use of `aws-auth` configmap with EKS cluster access entry (#2858) + +### Features + +* Replace the use of `aws-auth` configmap with EKS cluster access entry ([#2858](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2858)) ([6b40bdb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6b40bdbb1d283d9259f43b03d24dca99cc1eceff)) + ## [19.21.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.20.0...v19.21.0) (2023-12-11) From 417791374cf72dfb673105359463398eb4a75d6e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 3 Feb 2024 06:47:15 -0500 Subject: [PATCH 724/962] fix: Correct cluster access entry to create multiple policy associations per access entry (#2892) --- README.md | 34 ++++++++++++ examples/eks_managed_node_group/README.md | 1 + examples/eks_managed_node_group/main.tf | 65 +++++++++++++++++++++++ main.tf | 14 ++--- 4 files changed, 104 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 1039fa6c82..e75a816d11 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,12 @@ Terraform module which creates AWS EKS (Kubernetes) resources - [Upgrade to v17.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-17.0.md) - [Upgrade to v18.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-18.0.md) - [Upgrade to v19.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-19.0.md) + - [Upgrade to v20.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-19.0.md) ### External Documentation Please note that we strive to provide a comprehensive suite of documentation for __*configuring and utilizing the module(s)*__ defined here, and that documentation regarding EKS (including EKS managed node group, self managed node group, and Fargate profile) and/or Kubernetes features, usage, etc. are better left up to their respective sources: + - [AWS EKS Documentation](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Kubernetes Documentation](https://kubernetes.io/docs/home/) @@ -72,6 +74,28 @@ module "eks" { } } + # Cluster access entry + # To add the current caller identity as an administrator + enable_cluster_creator_admin_permissions = true + + access_entries = { + # One access entry with a policy associated + example = { + kubernetes_groups = [] + principal_arn = "arn:aws:iam::123456789012:role/something" + + policy_associations = { + example = { + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy" + access_scope = { + namespaces = ["default"] + type = "namespace" + } + } + } + } + } + tags = { Environment = "dev" Terraform = "true" @@ -79,6 +103,16 @@ module "eks" { } ``` +### Cluster Access Entry + +When enabling `authentication_mode = "API_AND_CONFIG_MAP"`, EKS will automatically create an access entry for the IAM role(s) used by managed nodegroup(s) and Fargate profile(s). There are no additional actions required by users. For self-managed nodegroups and the Karpenter sub-module, this project automatically adds the access entry on behalf of users so there are no additional actions required by users. + +On clusters that were created prior to CAM support, there will be an existing access entry for the cluster creator. This was previously not visible when using `aws-auth` ConfigMap, but will become visible when access entry is enabled. + +### Bootstrap Cluster Creator Admin Permissions + +Setting the `bootstrap_cluster_creator_admin_permissions` is a one time operation when the cluster is created; it cannot be modified later through the EKS API. In this project we are hardcoding this to `false`. If users wish to achieve the same functionality, we will do that through an access entry which can be enabled or disabled at any time of their choosing using the variable `enable_cluster_creator_admin_permissions` + ## Examples - [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group): EKS Cluster using EKS managed node groups diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 103d133553..e94558cd77 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -55,6 +55,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| | [aws_iam_policy.node_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_security_group.remote_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_ami.eks_default_arm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 0d17827fd8..de737d70fb 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -35,6 +35,8 @@ module "eks" { cluster_ip_family = "ipv6" create_cni_ipv6_iam_policy = true + enable_cluster_creator_admin_permissions = true + cluster_addons = { coredns = { most_recent = true @@ -241,6 +243,46 @@ module "eks" { } } + access_entries = { + # One access entry with a policy associated + ex-single = { + kubernetes_groups = [] + principal_arn = aws_iam_role.this["single"].arn + + policy_associations = { + single = { + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy" + access_scope = { + namespaces = ["default"] + type = "namespace" + } + } + } + } + + # Example of adding multiple policies to a single access entry + ex-multiple = { + kubernetes_groups = [] + principal_arn = aws_iam_role.this["multiple"].arn + + policy_associations = { + ex-one = { + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSEditPolicy" + access_scope = { + namespaces = ["default"] + type = "namespace" + } + } + ex-two = { + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy" + access_scope = { + type = "cluster" + } + } + } + } + } + tags = local.tags } @@ -436,3 +478,26 @@ data "aws_ami" "eks_default_bottlerocket" { values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"] } } + +resource "aws_iam_role" "this" { + for_each = toset(["single", "multiple"]) + + name = "ex-${each.key}" + + # Just using for this example + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "Example" + Principal = { + Service = "ec2.amazonaws.com" + } + }, + ] + }) + + tags = local.tags +} diff --git a/main.tf b/main.tf index a934c0f38e..8a2317a737 100644 --- a/main.tf +++ b/main.tf @@ -168,28 +168,22 @@ locals { for pol_key, pol_val in lookup(entry_val, "policy_associations", {}) : merge( { - principal_arn = entry_val.principal_arn - kubernetes_groups = lookup(entry_val, "kubernetes_groups", []) - tags = lookup(entry_val, "tags", {}) - type = lookup(entry_val, "type", "STANDARD") - user_name = lookup(entry_val, "user_name", null) + principal_arn = entry_val.principal_arn + entry_key = entry_key + pol_key = pol_key }, { for k, v in { association_policy_arn = pol_val.policy_arn association_access_scope_type = pol_val.access_scope.type association_access_scope_namespaces = lookup(pol_val.access_scope, "namespaces", []) } : k => v if !contains(["EC2_LINUX", "EC2_WINDOWS", "FARGATE_LINUX"], lookup(entry_val, "type", "STANDARD")) }, - { - entry_key = entry_key - pol_key = pol_key - } ) ] ]) } resource "aws_eks_access_entry" "this" { - for_each = { for k, v in local.flattened_access_entries : "${v.entry_key}_${v.pol_key}" => v if local.create } + for_each = { for k, v in local.merged_access_entries : k => v if local.create } cluster_name = aws_eks_cluster.this[0].name kubernetes_groups = try(each.value.kubernetes_groups, []) From 0b44d616f3e07b8d44f69860552310b8315929c8 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 3 Feb 2024 11:47:44 +0000 Subject: [PATCH 725/962] chore(release): version 20.0.1 [skip ci] ### [20.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.0.0...v20.0.1) (2024-02-03) ### Bug Fixes * Correct cluster access entry to create multiple policy associations per access entry ([#2892](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2892)) ([4177913](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/417791374cf72dfb673105359463398eb4a75d6e)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e7542610d..0a1dded341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [20.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.0.0...v20.0.1) (2024-02-03) + + +### Bug Fixes + +* Correct cluster access entry to create multiple policy associations per access entry ([#2892](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2892)) ([4177913](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/417791374cf72dfb673105359463398eb4a75d6e)) + ## [20.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v19.21.0...v20.0.0) (2024-02-02) From f8d49824b044f3ab9955822f029f5f1b7f2c0601 Mon Sep 17 00:00:00 2001 From: yutachaos <18604471+yutachaos@users.noreply.github.com> Date: Mon, 5 Feb 2024 21:57:49 +0900 Subject: [PATCH 726/962] docs: Correct link for `UPGRADE-20.0.md` on main README (#2898) Fixed link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e75a816d11..008fe78456 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Terraform module which creates AWS EKS (Kubernetes) resources - [Upgrade to v17.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-17.0.md) - [Upgrade to v18.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-18.0.md) - [Upgrade to v19.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-19.0.md) - - [Upgrade to v20.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-19.0.md) + - [Upgrade to v20.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-20.0.md) ### External Documentation From 3dbd1ec7822318578049bca7dbde83a3f3e0bafd Mon Sep 17 00:00:00 2001 From: an-dr-eas-k <16319480+an-dr-eas-k@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:07:21 +0100 Subject: [PATCH 727/962] docs: Correct resolve conflicts argument in `UPGRADE-20.0.md` (#2899) fix docu Co-authored-by: Bryant Biggs --- docs/UPGRADE-20.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/UPGRADE-20.0.md b/docs/UPGRADE-20.0.md index 72bdd57699..e273db3243 100644 --- a/docs/UPGRADE-20.0.md +++ b/docs/UPGRADE-20.0.md @@ -6,7 +6,7 @@ Please consult the `examples` directory for reference example configurations. If - Minium supported AWS provider version increased to `v5.34` - Minimum supported Terraform version increased to `v1.3` to support Terraform state `moved` blocks as well as other advanced features -- The `resolve_conflicts` argument within the `cluster_addons` configuration has been replaced with `resolve_conflicts_on_create` and `resolve_conflicts_on_delete` now that `resolve_conflicts` is deprecated +- The `resolve_conflicts` argument within the `cluster_addons` configuration has been replaced with `resolve_conflicts_on_create` and `resolve_conflicts_on_update` now that `resolve_conflicts` is deprecated - The default/fallback value for the `preserve` argument of `cluster_addons`is now set to `true`. This has shown to be useful for users deprovisioning clusters while avoiding the situation where the CNI is deleted too early and causes resources to be left orphaned resulting in conflicts. - The Karpenter sub-module's use of the `irsa` naming convention has been removed, along with an update to the Karpenter controller IAM policy to align with Karpenter's `v1beta1`/`v0.32` changes. Instead of referring to the role as `irsa` or `pod_identity`, its simply just an IAM role used by the Karpenter controller and there is support for use with either IRSA and/or Pod Identity (default) at this time - The `aws-auth` ConfigMap resources have been moved to a standalone sub-module. This removes the Kubernetes provider requirement from the main module and allows for the `aws-auth` ConfigMap to be managed independently of the main module. This sub-module will be removed entirely in the next major release. From 0381503cbbd953c913268bfa77201707424289a6 Mon Sep 17 00:00:00 2001 From: Nihal <37419479+its-nihal@users.noreply.github.com> Date: Tue, 6 Feb 2024 01:27:05 +0530 Subject: [PATCH 728/962] docs: Correct Karpenter variable references for changes made in `v20.0` (#2900) --- modules/karpenter/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 3262f71c63..e3082158e9 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -66,8 +66,8 @@ module "karpenter" { cluster_name = module.eks.cluster_name - create_iam_role = false - iam_role_arn = module.eks.eks_managed_node_groups["initial"].iam_role_arn + create_node_iam_role = false + node_iam_role_arn = module.eks.eks_managed_node_groups["initial"].iam_role_arn # Since the nodegroup role will already have an access entry create_access_entry = false From 0d2a4c2af3d7c8593226bbccbf8753950e741b15 Mon Sep 17 00:00:00 2001 From: Zack A <24322023+zack-is-cool@users.noreply.github.com> Date: Tue, 6 Feb 2024 04:48:19 -0800 Subject: [PATCH 729/962] feat: Add output for `access_policy_associations` (#2904) * add output for access_policy_associations * wording --- README.md | 1 + outputs.tf | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 008fe78456..7c6c65a421 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Description | |------|-------------| | [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | +| [access\_policy\_associations](#output\_access\_policy\_associations) | Map of eks cluster access policy associations created and their attributes | | [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | | [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | diff --git a/outputs.tf b/outputs.tf index adba15c10c..7621ae8cfa 100644 --- a/outputs.tf +++ b/outputs.tf @@ -61,6 +61,11 @@ output "access_entries" { value = aws_eks_access_entry.this } +output "access_policy_associations" { + description = "Map of eks cluster access policy associations created and their attributes" + value = aws_eks_access_policy_association.this +} + ################################################################################ # KMS Key ################################################################################ From 39f708c151876d5415d680738051b36acfd01330 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 6 Feb 2024 12:48:47 +0000 Subject: [PATCH 730/962] chore(release): version 20.1.0 [skip ci] ## [20.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.0.1...v20.1.0) (2024-02-06) ### Features * Add output for `access_policy_associations` ([#2904](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2904)) ([0d2a4c2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0d2a4c2af3d7c8593226bbccbf8753950e741b15)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a1dded341..038a56c94b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.0.1...v20.1.0) (2024-02-06) + + +### Features + +* Add output for `access_policy_associations` ([#2904](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2904)) ([0d2a4c2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0d2a4c2af3d7c8593226bbccbf8753950e741b15)) + ### [20.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.0.0...v20.0.1) (2024-02-03) From 1e32e6a9f8a389b1a4969dde697d34ba4e3c85ac Mon Sep 17 00:00:00 2001 From: Bourne-ID Date: Tue, 6 Feb 2024 09:04:22 -0500 Subject: [PATCH 731/962] fix: Update access entries `kubernetes_groups` default value to `null` (#2897) fix: Update default groups from [] to null Co-authored-by: Bryant Biggs --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 8a2317a737..f05f52959a 100644 --- a/main.tf +++ b/main.tf @@ -186,7 +186,7 @@ resource "aws_eks_access_entry" "this" { for_each = { for k, v in local.merged_access_entries : k => v if local.create } cluster_name = aws_eks_cluster.this[0].name - kubernetes_groups = try(each.value.kubernetes_groups, []) + kubernetes_groups = try(each.value.kubernetes_groups, null) principal_arn = each.value.principal_arn type = try(each.value.type, "STANDARD") user_name = try(each.value.user_name, null) From f6992b159cdfda954d44d40908ce5d06fc879703 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 6 Feb 2024 14:04:49 +0000 Subject: [PATCH 732/962] chore(release): version 20.1.1 [skip ci] ### [20.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.1.0...v20.1.1) (2024-02-06) ### Bug Fixes * Update access entries `kubernetes_groups` default value to `null` ([#2897](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2897)) ([1e32e6a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/1e32e6a9f8a389b1a4969dde697d34ba4e3c85ac)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 038a56c94b..58092abb6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [20.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.1.0...v20.1.1) (2024-02-06) + + +### Bug Fixes + +* Update access entries `kubernetes_groups` default value to `null` ([#2897](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2897)) ([1e32e6a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/1e32e6a9f8a389b1a4969dde697d34ba4e3c85ac)) + ## [20.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.0.1...v20.1.0) (2024-02-06) From cc6919de811f3972815d4ca26e5e0c8f64c2b894 Mon Sep 17 00:00:00 2001 From: tculp Date: Tue, 6 Feb 2024 09:34:04 -0500 Subject: [PATCH 733/962] feat: Allow enable/disable of EKS pod identity for the Karpenter controller (#2902) * Made EKS pod identities for the controller role toggleable * Switched the variable to the singular form --------- Co-authored-by: Tyler Culp --- modules/karpenter/README.md | 3 ++- modules/karpenter/main.tf | 20 ++++++++++++-------- modules/karpenter/variables.tf | 8 +++++++- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index e3082158e9..00f724acc5 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -135,7 +135,8 @@ No modules. | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created | `bool` | `true` | no | | [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `false` | no | | [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | -| [enable\_irsa](#input\_enable\_irsa) | Determines whether to enable support IAM role for service account | `bool` | `false` | no | +| [enable\_irsa](#input\_enable\_irsa) | Determines whether to enable support for IAM role for service accounts | `bool` | `false` | no | +| [enable\_pod\_identity](#input\_enable\_pod\_identity) | Determines whether to enable support for EKS pod identity | `bool` | `true` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | | [iam\_policy\_description](#input\_iam\_policy\_description) | IAM policy description | `string` | `"Karpenter controller IAM policy"` | no | | [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy | `string` | `"KarpenterController"` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index dfd6042e34..4fc3ef020a 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -22,15 +22,19 @@ data "aws_iam_policy_document" "controller_assume_role" { count = local.create_iam_role ? 1 : 0 # Pod Identity - statement { - actions = [ - "sts:AssumeRole", - "sts:TagSession", - ] + dynamic "statement" { + for_each = var.enable_pod_identity ? [1] : [] - principals { - type = "Service" - identifiers = ["pods.eks.amazonaws.com"] + content { + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "Service" + identifiers = ["pods.eks.amazonaws.com"] + } } } diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 3af82d4fc6..8d6bfe74ad 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -104,12 +104,18 @@ variable "ami_id_ssm_parameter_arns" { default = [] } +variable "enable_pod_identity" { + description = "Determines whether to enable support for EKS pod identity" + type = bool + default = true +} + ################################################################################ # IAM Role for Service Account (IRSA) ################################################################################ variable "enable_irsa" { - description = "Determines whether to enable support IAM role for service account" + description = "Determines whether to enable support for IAM role for service accounts" type = bool default = false } From a9212a423fc9e2959ac179c72a3c31bb5826e567 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 6 Feb 2024 14:34:33 +0000 Subject: [PATCH 734/962] chore(release): version 20.2.0 [skip ci] ## [20.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.1.1...v20.2.0) (2024-02-06) ### Features * Allow enable/disable of EKS pod identity for the Karpenter controller ([#2902](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2902)) ([cc6919d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/cc6919de811f3972815d4ca26e5e0c8f64c2b894)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58092abb6d..58d5183ac5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.1.1...v20.2.0) (2024-02-06) + + +### Features + +* Allow enable/disable of EKS pod identity for the Karpenter controller ([#2902](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2902)) ([cc6919d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/cc6919de811f3972815d4ca26e5e0c8f64c2b894)) + ### [20.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.1.0...v20.1.1) (2024-02-06) From 671fc6e627d957ada47ef3f33068d715e79d25d6 Mon Sep 17 00:00:00 2001 From: Nick Nikitas Date: Thu, 8 Feb 2024 02:32:47 +0200 Subject: [PATCH 735/962] fix: Karpenter `enable_spot_termination = false` should not result in an error (#2907) * Add condition for statement if enable_spot_termination is true * Update modules/karpenter/main.tf --------- Co-authored-by: Bryant Biggs --- modules/karpenter/main.tf | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 4fc3ef020a..add27954a8 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -252,15 +252,19 @@ data "aws_iam_policy_document" "controller" { actions = ["pricing:GetProducts"] } - statement { - sid = "AllowInterruptionQueueActions" - resources = [aws_sqs_queue.this[0].arn] - actions = [ - "sqs:DeleteMessage", - "sqs:GetQueueAttributes", - "sqs:GetQueueUrl", - "sqs:ReceiveMessage" - ] + dynamic "statement" { + for_each = local.enable_spot_termination ? [1] : [] + + content { + sid = "AllowInterruptionQueueActions" + resources = [try(aws_sqs_queue.this[0].arn, null)] + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] + } } statement { From 6c168effbe06f6e5ad4b122c7a3b491a2e1c874a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 8 Feb 2024 00:33:20 +0000 Subject: [PATCH 736/962] chore(release): version 20.2.1 [skip ci] ### [20.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.2.0...v20.2.1) (2024-02-08) ### Bug Fixes * Karpenter `enable_spot_termination = false` should not result in an error ([#2907](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2907)) ([671fc6e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/671fc6e627d957ada47ef3f33068d715e79d25d6)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58d5183ac5..8296b96869 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [20.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.2.0...v20.2.1) (2024-02-08) + + +### Bug Fixes + +* Karpenter `enable_spot_termination = false` should not result in an error ([#2907](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2907)) ([671fc6e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/671fc6e627d957ada47ef3f33068d715e79d25d6)) + ## [20.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.1.1...v20.2.0) (2024-02-06) From 659401d9bf277d50a2b3f9caca95852439dc3cb2 Mon Sep 17 00:00:00 2001 From: Stefano Zanella Date: Sun, 11 Feb 2024 14:44:42 +0100 Subject: [PATCH 737/962] docs: Lowercase Fargate compute type annotation value (#2922) --- examples/fargate_profile/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index 0b3c6b46c2..16fe82c1bc 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -35,7 +35,7 @@ module "eks" { vpc-cni = {} coredns = { configuration_values = jsonencode({ - computeType = "Fargate" + computeType = "fargate" }) } } From d6d97519d472561e6e6e0ca1b55e15cc321bc863 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 12 Feb 2024 16:34:23 -0500 Subject: [PATCH 738/962] docs: Update Karpenter example version; add additional clarification on access entry migration to upgrade guide (#2924) --- docs/UPGRADE-20.0.md | 17 ++++++++++++----- examples/karpenter/main.tf | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/UPGRADE-20.0.md b/docs/UPGRADE-20.0.md index e273db3243..5ae1b35783 100644 --- a/docs/UPGRADE-20.0.md +++ b/docs/UPGRADE-20.0.md @@ -221,6 +221,11 @@ Changing the `authentication_mode` is a one-way decision. See [announcement blog > Switching authentication modes on an existing cluster is a one-way operation. You can switch from CONFIG_MAP to API_AND_CONFIG_MAP. You can then switch from API_AND_CONFIG_MAP to API. You cannot revert these operations in the opposite direction. Meaning you cannot switch back to CONFIG_MAP or API_AND_CONFIG_MAP from API. And you cannot switch back to CONFIG_MAP from API_AND_CONFIG_MAP. +> [!IMPORTANT] +> If migrating to cluster access entries and you will NOT have any entries that remain in the `aws-auth` ConfigMap, you do not need to remove the configmap from the statefile. You can simply follow the migration guide and once access entries have been created, you can let Terraform remove/delete the `aws-auth` ConfigMap. +> +> If you WILL have entries that remain in the `aws-auth` ConfigMap, then you will need to remove the ConfigMap resources from the statefile to avoid any disruptions. When you add the new `aws-auth` sub-module and apply the changes, the sub-module will upsert the ConfigMap on the cluster. Provided the necessary entries are defined in that sub-module's definition, it will "re-adopt" the ConfigMap under Terraform's control. + ### authentication_mode = "CONFIG_MAP" If using `authentication_mode = "CONFIG_MAP"`, before making any changes, you will first need to remove the configmap from the statefile to avoid any disruptions: @@ -232,11 +237,10 @@ terraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]' # include if T Once the configmap has been removed from the statefile, you can add the new `aws-auth` sub-module and copy the relevant definitions from the EKS module over to the new `aws-auth` sub-module definition (see before after diff above). -#### ⚠️ Node IAM Roles - -You will need to add entries for any IAM roles used by nodegroups and/or Fargate profiles - the module no longer handles this in the background on behalf of users. - -When you apply the changes with the new sub-module, the configmap in the cluster will get updated with the contents provided in the sub-module definition, so please be sure all of the necessary entries are added before applying the changes. +> [!CAUTION] +> You will need to add entries to the `aws-auth` sub-module for any IAM roles used by nodegroups and/or Fargate profiles - the module no longer handles this in the background on behalf of users. +> +> When you apply the changes with the new sub-module, the configmap in the cluster will get updated with the contents provided in the sub-module definition, so please be sure all of the necessary entries are added before applying the changes. ### authentication_mode = "API_AND_CONFIG_MAP" @@ -244,6 +248,9 @@ When using `authentication_mode = "API_AND_CONFIG_MAP"` and there are entries th Once the `authentication_mode` has been updated, next you will need to remove the configmap from the statefile to avoid any disruptions: +> [!NOTE] +> This is only required if there are entries that will remain in the `aws-auth` ConfigMap after migrating. Otherwise, you can skip this step and let Terraform destroy the ConfigMap. + ```sh terraform state rm 'module.eks.kubernetes_config_map_v1_data.aws_auth[0]' terraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]' # include if Terraform created the original configmap diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index cd36785893..bd1540b3a0 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -42,7 +42,7 @@ data "aws_ecrpublic_authorization_token" "token" { locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.28" + cluster_version = "1.29" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -169,7 +169,7 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "v0.33.1" + version = "v0.34.0" wait = false values = [ From d63fd61e892024feb1de09d96ef9a18c764bcec8 Mon Sep 17 00:00:00 2001 From: Mukund Tiwari <88654449+mukund-tiwari@users.noreply.github.com> Date: Tue, 20 Feb 2024 18:20:06 +0530 Subject: [PATCH 739/962] docs: Correct mis-spelling in README (#2935) --- examples/karpenter/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index b2c42931dd..a1303999b0 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -42,7 +42,7 @@ kubectl delete node -l karpenter.sh/provisioner-name=default ```bash # Necessary to avoid removing Terraform's permissions too soon before its finished -# cleaning up the resources it deployed inside the clsuter +# cleaning up the resources it deployed inside the cluster terraform state rm 'module.eks.aws_eks_access_entry.this["cluster_creator_admin"]' || true terraform destroy ``` From 081c7624a5a4f2b039370ae8eb9ee8e445d01c48 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 21 Feb 2024 17:30:49 -0500 Subject: [PATCH 740/962] fix: Replace Karpenter SQS policy dynamic service princpal DNS suffixes with static `amazonaws.com` (#2941) fix: Replace dynamic service princpal DNS suffixes with static `amazonaws.com` --- .pre-commit-config.yaml | 2 +- modules/karpenter/main.tf | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 93900a8e02..9af61e655b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.86.0 + rev: v1.87.1 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index add27954a8..4d01d77a01 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -4,7 +4,6 @@ data "aws_caller_identity" "current" {} locals { account_id = data.aws_caller_identity.current.account_id - dns_suffix = data.aws_partition.current.dns_suffix partition = data.aws_partition.current.partition region = data.aws_region.current.name } @@ -445,8 +444,8 @@ data "aws_iam_policy_document" "queue" { principals { type = "Service" identifiers = [ - "events.${local.dns_suffix}", - "sqs.${local.dns_suffix}", + "events.amazonaws.com", + "sqs.amazonaws.com", ] } } From 015475430a0ec79ee2912b203287c1732889e3c3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 21 Feb 2024 22:31:19 +0000 Subject: [PATCH 741/962] chore(release): version 20.2.2 [skip ci] ### [20.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.2.1...v20.2.2) (2024-02-21) ### Bug Fixes * Replace Karpenter SQS policy dynamic service princpal DNS suffixes with static `amazonaws.com` ([#2941](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2941)) ([081c762](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/081c7624a5a4f2b039370ae8eb9ee8e445d01c48)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8296b96869..94178d1342 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [20.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.2.1...v20.2.2) (2024-02-21) + + +### Bug Fixes + +* Replace Karpenter SQS policy dynamic service princpal DNS suffixes with static `amazonaws.com` ([#2941](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2941)) ([081c762](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/081c7624a5a4f2b039370ae8eb9ee8e445d01c48)) + ### [20.2.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.2.0...v20.2.1) (2024-02-08) From 0d468bd432c571a002b5578ee08eca2dffe5fc3e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 21 Feb 2024 17:32:01 -0500 Subject: [PATCH 742/962] docs: Update output for user data example to capture rendered output (#2940) * docs: Update output for user data example to capture renereded output * fix: Tsk tsk tsk - I forgot to run pre-commit --- examples/user_data/README.md | 67 +++---- examples/user_data/main.tf | 42 +++-- examples/user_data/outputs.tf | 164 ++++++++++-------- .../rendered/al2/eks-mng-additional.sh | 11 ++ .../rendered/al2/eks-mng-custom-ami.sh | 8 + .../rendered/al2/eks-mng-custom-template.sh | 11 ++ .../user_data/rendered/al2/eks-mng-no-op.sh | 0 .../rendered/al2/self-mng-bootstrap.sh | 8 + .../rendered/al2/self-mng-custom-template.sh | 11 ++ .../user_data/rendered/al2/self-mng-no-op.sh | 0 .../bottlerocket/eks-mng-additional.toml | 3 + .../bottlerocket/eks-mng-custom-ami.toml | 7 + .../bottlerocket/eks-mng-custom-template.toml | 9 + .../rendered/bottlerocket/eks-mng-no-op.toml | 0 .../bottlerocket/self-mng-bootstrap.toml | 7 + .../self-mng-custom-template.toml | 9 + .../rendered/bottlerocket/self-mng-no-op.toml | 0 .../rendered/windows/eks-mng-additional.ps1 | 1 + .../rendered/windows/eks-mng-custom-ami.ps1 | 9 + .../windows/eks-mng-custom-template.ps1 | 10 ++ .../rendered/windows/eks-mng-no-op.ps1 | 0 .../rendered/windows/self-mng-bootstrap.ps1 | 9 + .../windows/self-mng-custom-template.ps1 | 10 ++ .../rendered/windows/self-mng-no-op.ps1 | 0 examples/user_data/versions.tf | 7 + 25 files changed, 288 insertions(+), 115 deletions(-) create mode 100755 examples/user_data/rendered/al2/eks-mng-additional.sh create mode 100755 examples/user_data/rendered/al2/eks-mng-custom-ami.sh create mode 100755 examples/user_data/rendered/al2/eks-mng-custom-template.sh create mode 100755 examples/user_data/rendered/al2/eks-mng-no-op.sh create mode 100755 examples/user_data/rendered/al2/self-mng-bootstrap.sh create mode 100755 examples/user_data/rendered/al2/self-mng-custom-template.sh create mode 100755 examples/user_data/rendered/al2/self-mng-no-op.sh create mode 100755 examples/user_data/rendered/bottlerocket/eks-mng-additional.toml create mode 100755 examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml create mode 100755 examples/user_data/rendered/bottlerocket/eks-mng-custom-template.toml create mode 100755 examples/user_data/rendered/bottlerocket/eks-mng-no-op.toml create mode 100755 examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml create mode 100755 examples/user_data/rendered/bottlerocket/self-mng-custom-template.toml create mode 100755 examples/user_data/rendered/bottlerocket/self-mng-no-op.toml create mode 100755 examples/user_data/rendered/windows/eks-mng-additional.ps1 create mode 100755 examples/user_data/rendered/windows/eks-mng-custom-ami.ps1 create mode 100755 examples/user_data/rendered/windows/eks-mng-custom-template.ps1 create mode 100755 examples/user_data/rendered/windows/eks-mng-no-op.ps1 create mode 100755 examples/user_data/rendered/windows/self-mng-bootstrap.ps1 create mode 100755 examples/user_data/rendered/windows/self-mng-custom-template.ps1 create mode 100755 examples/user_data/rendered/windows/self-mng-no-op.ps1 diff --git a/examples/user_data/README.md b/examples/user_data/README.md index b2215a02bb..6e1605ca1c 100644 --- a/examples/user_data/README.md +++ b/examples/user_data/README.md @@ -18,40 +18,65 @@ $ terraform apply | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | +| [local](#requirement\_local) | >= 2.4 | ## Providers -No providers. +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.4 | ## Modules | Name | Source | Version | |------|--------|---------| +| [eks\_mng\_al2\_additional](#module\_eks\_mng\_al2\_additional) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2\_custom\_ami](#module\_eks\_mng\_al2\_custom\_ami) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2\_custom\_template](#module\_eks\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2\_no\_op](#module\_eks\_mng\_al2\_no\_op) | ../../modules/_user_data | n/a | | [eks\_mng\_bottlerocket\_additional](#module\_eks\_mng\_bottlerocket\_additional) | ../../modules/_user_data | n/a | | [eks\_mng\_bottlerocket\_custom\_ami](#module\_eks\_mng\_bottlerocket\_custom\_ami) | ../../modules/_user_data | n/a | | [eks\_mng\_bottlerocket\_custom\_template](#module\_eks\_mng\_bottlerocket\_custom\_template) | ../../modules/_user_data | n/a | | [eks\_mng\_bottlerocket\_no\_op](#module\_eks\_mng\_bottlerocket\_no\_op) | ../../modules/_user_data | n/a | -| [eks\_mng\_linux\_additional](#module\_eks\_mng\_linux\_additional) | ../../modules/_user_data | n/a | -| [eks\_mng\_linux\_custom\_ami](#module\_eks\_mng\_linux\_custom\_ami) | ../../modules/_user_data | n/a | -| [eks\_mng\_linux\_custom\_template](#module\_eks\_mng\_linux\_custom\_template) | ../../modules/_user_data | n/a | -| [eks\_mng\_linux\_no\_op](#module\_eks\_mng\_linux\_no\_op) | ../../modules/_user_data | n/a | | [eks\_mng\_windows\_additional](#module\_eks\_mng\_windows\_additional) | ../../modules/_user_data | n/a | | [eks\_mng\_windows\_custom\_ami](#module\_eks\_mng\_windows\_custom\_ami) | ../../modules/_user_data | n/a | | [eks\_mng\_windows\_custom\_template](#module\_eks\_mng\_windows\_custom\_template) | ../../modules/_user_data | n/a | | [eks\_mng\_windows\_no\_op](#module\_eks\_mng\_windows\_no\_op) | ../../modules/_user_data | n/a | +| [self\_mng\_al2\_bootstrap](#module\_self\_mng\_al2\_bootstrap) | ../../modules/_user_data | n/a | +| [self\_mng\_al2\_custom\_template](#module\_self\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a | +| [self\_mng\_al2\_no\_op](#module\_self\_mng\_al2\_no\_op) | ../../modules/_user_data | n/a | | [self\_mng\_bottlerocket\_bootstrap](#module\_self\_mng\_bottlerocket\_bootstrap) | ../../modules/_user_data | n/a | | [self\_mng\_bottlerocket\_custom\_template](#module\_self\_mng\_bottlerocket\_custom\_template) | ../../modules/_user_data | n/a | | [self\_mng\_bottlerocket\_no\_op](#module\_self\_mng\_bottlerocket\_no\_op) | ../../modules/_user_data | n/a | -| [self\_mng\_linux\_bootstrap](#module\_self\_mng\_linux\_bootstrap) | ../../modules/_user_data | n/a | -| [self\_mng\_linux\_custom\_template](#module\_self\_mng\_linux\_custom\_template) | ../../modules/_user_data | n/a | -| [self\_mng\_linux\_no\_op](#module\_self\_mng\_linux\_no\_op) | ../../modules/_user_data | n/a | | [self\_mng\_windows\_bootstrap](#module\_self\_mng\_windows\_bootstrap) | ../../modules/_user_data | n/a | | [self\_mng\_windows\_custom\_template](#module\_self\_mng\_windows\_custom\_template) | ../../modules/_user_data | n/a | | [self\_mng\_windows\_no\_op](#module\_self\_mng\_windows\_no\_op) | ../../modules/_user_data | n/a | ## Resources -No resources. +| Name | Type | +|------|------| +| [local_file.eks_mng_al2_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_al2_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_al2_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_bottlerocket_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_bottlerocket_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_bottlerocket_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_bottlerocket_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_windows_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_windows_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_windows_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_windows_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_al2_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_al2_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_bottlerocket_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_bottlerocket_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_bottlerocket_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_windows_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_windows_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_windows_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | ## Inputs @@ -59,27 +84,5 @@ No inputs. ## Outputs -| Name | Description | -|------|-------------| -| [eks\_mng\_bottlerocket\_additional](#output\_eks\_mng\_bottlerocket\_additional) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_bottlerocket\_custom\_ami](#output\_eks\_mng\_bottlerocket\_custom\_ami) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_bottlerocket\_custom\_template](#output\_eks\_mng\_bottlerocket\_custom\_template) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_bottlerocket\_no\_op](#output\_eks\_mng\_bottlerocket\_no\_op) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_linux\_additional](#output\_eks\_mng\_linux\_additional) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_linux\_custom\_ami](#output\_eks\_mng\_linux\_custom\_ami) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_linux\_custom\_template](#output\_eks\_mng\_linux\_custom\_template) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_linux\_no\_op](#output\_eks\_mng\_linux\_no\_op) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_windows\_additional](#output\_eks\_mng\_windows\_additional) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_windows\_custom\_ami](#output\_eks\_mng\_windows\_custom\_ami) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_windows\_custom\_template](#output\_eks\_mng\_windows\_custom\_template) | Base64 decoded user data rendered for the provided inputs | -| [eks\_mng\_windows\_no\_op](#output\_eks\_mng\_windows\_no\_op) | Base64 decoded user data rendered for the provided inputs | -| [self\_mng\_bottlerocket\_bootstrap](#output\_self\_mng\_bottlerocket\_bootstrap) | Base64 decoded user data rendered for the provided inputs | -| [self\_mng\_bottlerocket\_custom\_template](#output\_self\_mng\_bottlerocket\_custom\_template) | Base64 decoded user data rendered for the provided inputs | -| [self\_mng\_bottlerocket\_no\_op](#output\_self\_mng\_bottlerocket\_no\_op) | Base64 decoded user data rendered for the provided inputs | -| [self\_mng\_linux\_bootstrap](#output\_self\_mng\_linux\_bootstrap) | Base64 decoded user data rendered for the provided inputs | -| [self\_mng\_linux\_custom\_template](#output\_self\_mng\_linux\_custom\_template) | Base64 decoded user data rendered for the provided inputs | -| [self\_mng\_linux\_no\_op](#output\_self\_mng\_linux\_no\_op) | Base64 decoded user data rendered for the provided inputs | -| [self\_mng\_windows\_bootstrap](#output\_self\_mng\_windows\_bootstrap) | Base64 decoded user data rendered for the provided inputs | -| [self\_mng\_windows\_custom\_template](#output\_self\_mng\_windows\_custom\_template) | Base64 decoded user data rendered for the provided inputs | -| [self\_mng\_windows\_no\_op](#output\_self\_mng\_windows\_no\_op) | Base64 decoded user data rendered for the provided inputs | +No outputs. diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index ea08c8c6d0..58536357d5 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -7,15 +7,14 @@ locals { } ################################################################################ -# User Data Module +# EKS managed node group - AL2 ################################################################################ -# EKS managed node group - linux -module "eks_mng_linux_no_op" { +module "eks_mng_al2_no_op" { source = "../../modules/_user_data" } -module "eks_mng_linux_additional" { +module "eks_mng_al2_additional" { source = "../../modules/_user_data" pre_bootstrap_user_data = <<-EOT @@ -23,7 +22,7 @@ module "eks_mng_linux_additional" { EOT } -module "eks_mng_linux_custom_ami" { +module "eks_mng_al2_custom_ami" { source = "../../modules/_user_data" cluster_name = local.name @@ -45,7 +44,7 @@ module "eks_mng_linux_custom_ami" { } -module "eks_mng_linux_custom_template" { +module "eks_mng_al2_custom_template" { source = "../../modules/_user_data" cluster_name = local.name @@ -66,7 +65,10 @@ module "eks_mng_linux_custom_template" { EOT } -# EKS managed node group - bottlerocket +################################################################################ +# EKS managed node group - Bottlerocket +################################################################################ + module "eks_mng_bottlerocket_no_op" { source = "../../modules/_user_data" @@ -121,7 +123,10 @@ module "eks_mng_bottlerocket_custom_template" { EOT } -# EKS managed node group - windows +################################################################################ +# EKS managed node group - Windows +################################################################################ + module "eks_mng_windows_no_op" { source = "../../modules/_user_data" @@ -184,14 +189,17 @@ module "eks_mng_windows_custom_template" { EOT } -# Self managed node group - linux -module "self_mng_linux_no_op" { +################################################################################ +# Self-managed node group - AL2 +################################################################################ + +module "self_mng_al2_no_op" { source = "../../modules/_user_data" is_eks_managed_node_group = false } -module "self_mng_linux_bootstrap" { +module "self_mng_al2_bootstrap" { source = "../../modules/_user_data" enable_bootstrap_user_data = true @@ -213,7 +221,7 @@ module "self_mng_linux_bootstrap" { EOT } -module "self_mng_linux_custom_template" { +module "self_mng_al2_custom_template" { source = "../../modules/_user_data" enable_bootstrap_user_data = true @@ -237,7 +245,10 @@ module "self_mng_linux_custom_template" { EOT } -# Self managed node group - bottlerocket +################################################################################ +# Self-managed node group - Bottlerocket +################################################################################ + module "self_mng_bottlerocket_no_op" { source = "../../modules/_user_data" @@ -286,7 +297,10 @@ module "self_mng_bottlerocket_custom_template" { EOT } -# Self managed node group - windows +################################################################################ +# Self-managed node group - Windows +################################################################################ + module "self_mng_windows_no_op" { source = "../../modules/_user_data" diff --git a/examples/user_data/outputs.tf b/examples/user_data/outputs.tf index 768d81c37a..22988e9d43 100644 --- a/examples/user_data/outputs.tf +++ b/examples/user_data/outputs.tf @@ -1,110 +1,136 @@ -# EKS managed node group - linux -output "eks_mng_linux_no_op" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_linux_no_op.user_data) +################################################################################ +# We are writing to local file so that we can better track diffs across changes +# +# Its harder to verify changes and diffs when we use the standard `output` +# route, writing to file makes this easier and better highlights changes +# to avoid unintended disruptions +################################################################################ + +################################################################################ +# EKS managed node group - AL2 +################################################################################ + +resource "local_file" "eks_mng_al2_no_op" { + content = base64decode(module.eks_mng_al2_no_op.user_data) + filename = "${path.module}/rendered/al2/eks-mng-no-op.sh" } -output "eks_mng_linux_additional" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_linux_additional.user_data) +resource "local_file" "eks_mng_al2_additional" { + content = base64decode(module.eks_mng_al2_additional.user_data) + filename = "${path.module}/rendered/al2/eks-mng-additional.sh" } -output "eks_mng_linux_custom_ami" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_linux_custom_ami.user_data) +resource "local_file" "eks_mng_al2_custom_ami" { + content = base64decode(module.eks_mng_al2_custom_ami.user_data) + filename = "${path.module}/rendered/al2/eks-mng-custom-ami.sh" } -output "eks_mng_linux_custom_template" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_linux_custom_template.user_data) +resource "local_file" "eks_mng_al2_custom_template" { + content = base64decode(module.eks_mng_al2_custom_template.user_data) + filename = "${path.module}/rendered/al2/eks-mng-custom-template.sh" } -# EKS managed node group - bottlerocket -output "eks_mng_bottlerocket_no_op" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_bottlerocket_no_op.user_data) +################################################################################ +# EKS managed node group - Bottlerocket +################################################################################ + +resource "local_file" "eks_mng_bottlerocket_no_op" { + content = base64decode(module.eks_mng_bottlerocket_no_op.user_data) + filename = "${path.module}/rendered/bottlerocket/eks-mng-no-op.toml" } -output "eks_mng_bottlerocket_additional" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_bottlerocket_additional.user_data) +resource "local_file" "eks_mng_bottlerocket_additional" { + content = base64decode(module.eks_mng_bottlerocket_additional.user_data) + filename = "${path.module}/rendered/bottlerocket/eks-mng-additional.toml" } -output "eks_mng_bottlerocket_custom_ami" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_bottlerocket_custom_ami.user_data) +resource "local_file" "eks_mng_bottlerocket_custom_ami" { + content = base64decode(module.eks_mng_bottlerocket_custom_ami.user_data) + filename = "${path.module}/rendered/bottlerocket/eks-mng-custom-ami.toml" } -output "eks_mng_bottlerocket_custom_template" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_bottlerocket_custom_template.user_data) +resource "local_file" "eks_mng_bottlerocket_custom_template" { + content = base64decode(module.eks_mng_bottlerocket_custom_template.user_data) + filename = "${path.module}/rendered/bottlerocket/eks-mng-custom-template.toml" } -# EKS managed node group - windows -output "eks_mng_windows_no_op" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_windows_no_op.user_data) +################################################################################ +# EKS managed node group - Windows +################################################################################ + +resource "local_file" "eks_mng_windows_no_op" { + content = base64decode(module.eks_mng_windows_no_op.user_data) + filename = "${path.module}/rendered/windows/eks-mng-no-op.ps1" } -output "eks_mng_windows_additional" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_windows_additional.user_data) +resource "local_file" "eks_mng_windows_additional" { + content = base64decode(module.eks_mng_windows_additional.user_data) + filename = "${path.module}/rendered/windows/eks-mng-additional.ps1" } -output "eks_mng_windows_custom_ami" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_windows_custom_ami.user_data) +resource "local_file" "eks_mng_windows_custom_ami" { + content = base64decode(module.eks_mng_windows_custom_ami.user_data) + filename = "${path.module}/rendered/windows/eks-mng-custom-ami.ps1" } -output "eks_mng_windows_custom_template" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.eks_mng_windows_custom_template.user_data) +resource "local_file" "eks_mng_windows_custom_template" { + content = base64decode(module.eks_mng_windows_custom_template.user_data) + filename = "${path.module}/rendered/windows/eks-mng-custom-template.ps1" } -# Self managed node group - linux -output "self_mng_linux_no_op" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.self_mng_linux_no_op.user_data) +################################################################################ +# Self-managed node group - AL2 +################################################################################ + +resource "local_file" "self_mng_al2_no_op" { + content = base64decode(module.self_mng_al2_no_op.user_data) + filename = "${path.module}/rendered/al2/self-mng-no-op.sh" } -output "self_mng_linux_bootstrap" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.self_mng_linux_bootstrap.user_data) +resource "local_file" "self_mng_al2_bootstrap" { + content = base64decode(module.self_mng_al2_bootstrap.user_data) + filename = "${path.module}/rendered/al2/self-mng-bootstrap.sh" } -output "self_mng_linux_custom_template" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.self_mng_linux_custom_template.user_data) +resource "local_file" "self_mng_al2_custom_template" { + content = base64decode(module.self_mng_al2_custom_template.user_data) + filename = "${path.module}/rendered/al2/self-mng-custom-template.sh" } -# Self managed node group - bottlerocket -output "self_mng_bottlerocket_no_op" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.self_mng_bottlerocket_no_op.user_data) +################################################################################ +# Self-managed node group - Bottlerocket +################################################################################ + +resource "local_file" "self_mng_bottlerocket_no_op" { + content = base64decode(module.self_mng_bottlerocket_no_op.user_data) + filename = "${path.module}/rendered/bottlerocket/self-mng-no-op.toml" } -output "self_mng_bottlerocket_bootstrap" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.self_mng_bottlerocket_bootstrap.user_data) +resource "local_file" "self_mng_bottlerocket_bootstrap" { + content = base64decode(module.self_mng_bottlerocket_bootstrap.user_data) + filename = "${path.module}/rendered/bottlerocket/self-mng-bootstrap.toml" } -output "self_mng_bottlerocket_custom_template" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.self_mng_bottlerocket_custom_template.user_data) +resource "local_file" "self_mng_bottlerocket_custom_template" { + content = base64decode(module.self_mng_bottlerocket_custom_template.user_data) + filename = "${path.module}/rendered/bottlerocket/self-mng-custom-template.toml" } -# Self managed node group - windows -output "self_mng_windows_no_op" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.self_mng_windows_no_op.user_data) +################################################################################ +# Self-managed node group - Windows +################################################################################ + +resource "local_file" "self_mng_windows_no_op" { + content = base64decode(module.self_mng_windows_no_op.user_data) + filename = "${path.module}/rendered/windows/self-mng-no-op.ps1" } -output "self_mng_windows_bootstrap" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.self_mng_windows_bootstrap.user_data) +resource "local_file" "self_mng_windows_bootstrap" { + content = base64decode(module.self_mng_windows_bootstrap.user_data) + filename = "${path.module}/rendered/windows/self-mng-bootstrap.ps1" } -output "self_mng_windows_custom_template" { - description = "Base64 decoded user data rendered for the provided inputs" - value = base64decode(module.self_mng_windows_custom_template.user_data) +resource "local_file" "self_mng_windows_custom_template" { + content = base64decode(module.self_mng_windows_custom_template.user_data) + filename = "${path.module}/rendered/windows/self-mng-custom-template.ps1" } diff --git a/examples/user_data/rendered/al2/eks-mng-additional.sh b/examples/user_data/rendered/al2/eks-mng-additional.sh new file mode 100755 index 0000000000..70b3142abb --- /dev/null +++ b/examples/user_data/rendered/al2/eks-mng-additional.sh @@ -0,0 +1,11 @@ +Content-Type: multipart/mixed; boundary="//" +MIME-Version: 1.0 + +--// +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript +Mime-Version: 1.0 + +export USE_MAX_PODS=false + +--//-- diff --git a/examples/user_data/rendered/al2/eks-mng-custom-ami.sh b/examples/user_data/rendered/al2/eks-mng-custom-ami.sh new file mode 100755 index 0000000000..4be9d969b4 --- /dev/null +++ b/examples/user_data/rendered/al2/eks-mng-custom-ami.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e +export FOO=bar +export SERVICE_IPV4_CIDR=172.16.0.0/16 +B64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== +API_SERVER_URL=https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com +/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--instance-type t3a.large' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +echo "All done" diff --git a/examples/user_data/rendered/al2/eks-mng-custom-template.sh b/examples/user_data/rendered/al2/eks-mng-custom-template.sh new file mode 100755 index 0000000000..d27a40a135 --- /dev/null +++ b/examples/user_data/rendered/al2/eks-mng-custom-template.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -ex + +echo "foo" +export FOO=bar + +# Custom user data template provided for rendering +B64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== +API_SERVER_URL=https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com +/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +echo "All done" diff --git a/examples/user_data/rendered/al2/eks-mng-no-op.sh b/examples/user_data/rendered/al2/eks-mng-no-op.sh new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/rendered/al2/self-mng-bootstrap.sh b/examples/user_data/rendered/al2/self-mng-bootstrap.sh new file mode 100755 index 0000000000..5dea4bf032 --- /dev/null +++ b/examples/user_data/rendered/al2/self-mng-bootstrap.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e +echo "foo" +export FOO=bar +B64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== +API_SERVER_URL=https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com +/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +echo "All done" diff --git a/examples/user_data/rendered/al2/self-mng-custom-template.sh b/examples/user_data/rendered/al2/self-mng-custom-template.sh new file mode 100755 index 0000000000..d27a40a135 --- /dev/null +++ b/examples/user_data/rendered/al2/self-mng-custom-template.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -ex + +echo "foo" +export FOO=bar + +# Custom user data template provided for rendering +B64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== +API_SERVER_URL=https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com +/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +echo "All done" diff --git a/examples/user_data/rendered/al2/self-mng-no-op.sh b/examples/user_data/rendered/al2/self-mng-no-op.sh new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/rendered/bottlerocket/eks-mng-additional.toml b/examples/user_data/rendered/bottlerocket/eks-mng-additional.toml new file mode 100755 index 0000000000..7ed4affaf6 --- /dev/null +++ b/examples/user_data/rendered/bottlerocket/eks-mng-additional.toml @@ -0,0 +1,3 @@ +# extra args added +[settings.kernel] +lockdown = "integrity" diff --git a/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml b/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml new file mode 100755 index 0000000000..a1bac20219 --- /dev/null +++ b/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml @@ -0,0 +1,7 @@ +[settings.kubernetes] +"cluster-name" = "ex-user-data" +"api-server" = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" +"cluster-certificate" = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" +# extra args added +[settings.kernel] +lockdown = "integrity" diff --git a/examples/user_data/rendered/bottlerocket/eks-mng-custom-template.toml b/examples/user_data/rendered/bottlerocket/eks-mng-custom-template.toml new file mode 100755 index 0000000000..c5c6774cfc --- /dev/null +++ b/examples/user_data/rendered/bottlerocket/eks-mng-custom-template.toml @@ -0,0 +1,9 @@ +# Custom user data template provided for rendering +[settings.kubernetes] +"cluster-name" = "ex-user-data" +"api-server" = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" +"cluster-certificate" = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" + +# extra args added +[settings.kernel] +lockdown = "integrity" diff --git a/examples/user_data/rendered/bottlerocket/eks-mng-no-op.toml b/examples/user_data/rendered/bottlerocket/eks-mng-no-op.toml new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml b/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml new file mode 100755 index 0000000000..a1bac20219 --- /dev/null +++ b/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml @@ -0,0 +1,7 @@ +[settings.kubernetes] +"cluster-name" = "ex-user-data" +"api-server" = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" +"cluster-certificate" = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" +# extra args added +[settings.kernel] +lockdown = "integrity" diff --git a/examples/user_data/rendered/bottlerocket/self-mng-custom-template.toml b/examples/user_data/rendered/bottlerocket/self-mng-custom-template.toml new file mode 100755 index 0000000000..c5c6774cfc --- /dev/null +++ b/examples/user_data/rendered/bottlerocket/self-mng-custom-template.toml @@ -0,0 +1,9 @@ +# Custom user data template provided for rendering +[settings.kubernetes] +"cluster-name" = "ex-user-data" +"api-server" = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" +"cluster-certificate" = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" + +# extra args added +[settings.kernel] +lockdown = "integrity" diff --git a/examples/user_data/rendered/bottlerocket/self-mng-no-op.toml b/examples/user_data/rendered/bottlerocket/self-mng-no-op.toml new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/rendered/windows/eks-mng-additional.ps1 b/examples/user_data/rendered/windows/eks-mng-additional.ps1 new file mode 100755 index 0000000000..0debfcf9ad --- /dev/null +++ b/examples/user_data/rendered/windows/eks-mng-additional.ps1 @@ -0,0 +1 @@ +[string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' diff --git a/examples/user_data/rendered/windows/eks-mng-custom-ami.ps1 b/examples/user_data/rendered/windows/eks-mng-custom-ami.ps1 new file mode 100755 index 0000000000..182195b707 --- /dev/null +++ b/examples/user_data/rendered/windows/eks-mng-custom-ami.ps1 @@ -0,0 +1,9 @@ + +[string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' +[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" +[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' +[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" +& $EKSBootstrapScriptFile -EKSClusterName ex-user-data -APIServerEndpoint https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com -Base64ClusterCA LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== -KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot 3>&1 4>&1 5>&1 6>&1 +$LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } +[string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + diff --git a/examples/user_data/rendered/windows/eks-mng-custom-template.ps1 b/examples/user_data/rendered/windows/eks-mng-custom-template.ps1 new file mode 100755 index 0000000000..aa4008c7e5 --- /dev/null +++ b/examples/user_data/rendered/windows/eks-mng-custom-template.ps1 @@ -0,0 +1,10 @@ +# Custom user data template provided for rendering + +[string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' +[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" +[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' +[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" +& $EKSBootstrapScriptFile -EKSClusterName ex-user-data -APIServerEndpoint https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com -Base64ClusterCA LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== -KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot 3>&1 4>&1 5>&1 6>&1 +$LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } +[string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + diff --git a/examples/user_data/rendered/windows/eks-mng-no-op.ps1 b/examples/user_data/rendered/windows/eks-mng-no-op.ps1 new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/rendered/windows/self-mng-bootstrap.ps1 b/examples/user_data/rendered/windows/self-mng-bootstrap.ps1 new file mode 100755 index 0000000000..182195b707 --- /dev/null +++ b/examples/user_data/rendered/windows/self-mng-bootstrap.ps1 @@ -0,0 +1,9 @@ + +[string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' +[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" +[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' +[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" +& $EKSBootstrapScriptFile -EKSClusterName ex-user-data -APIServerEndpoint https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com -Base64ClusterCA LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== -KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot 3>&1 4>&1 5>&1 6>&1 +$LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } +[string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + diff --git a/examples/user_data/rendered/windows/self-mng-custom-template.ps1 b/examples/user_data/rendered/windows/self-mng-custom-template.ps1 new file mode 100755 index 0000000000..aa4008c7e5 --- /dev/null +++ b/examples/user_data/rendered/windows/self-mng-custom-template.ps1 @@ -0,0 +1,10 @@ +# Custom user data template provided for rendering + +[string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' +[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" +[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' +[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" +& $EKSBootstrapScriptFile -EKSClusterName ex-user-data -APIServerEndpoint https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com -Base64ClusterCA LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== -KubeletExtraArgs --node-labels=node.kubernetes.io/lifecycle=spot 3>&1 4>&1 5>&1 6>&1 +$LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } +[string]$Something = 'IStillDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' + diff --git a/examples/user_data/rendered/windows/self-mng-no-op.ps1 b/examples/user_data/rendered/windows/self-mng-no-op.ps1 new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/versions.tf b/examples/user_data/versions.tf index 696426845c..af91653bb0 100644 --- a/examples/user_data/versions.tf +++ b/examples/user_data/versions.tf @@ -1,3 +1,10 @@ terraform { required_version = ">= 1.3" + + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.4" + } + } } From f6255c49e47d44bd62bb2b4e1e448ac80ceb2b3a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 21 Feb 2024 17:33:59 -0500 Subject: [PATCH 743/962] feat: Add support for addon and identity provider custom tags (#2938) feat: Add support for addon and OIDC provider custom tags --- main.tf | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/main.tf b/main.tf index f05f52959a..2f169936a0 100644 --- a/main.tf +++ b/main.tf @@ -481,6 +481,14 @@ resource "aws_iam_policy" "cluster_encryption" { # EKS Addons ################################################################################ +data "aws_eks_addon_version" "this" { + for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } + + addon_name = try(each.value.name, each.key) + kubernetes_version = coalesce(var.cluster_version, aws_eks_cluster.this[0].version) + most_recent = try(each.value.most_recent, null) +} + resource "aws_eks_addon" "this" { # Not supported on outposts for_each = { for k, v in var.cluster_addons : k => v if !try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } @@ -507,7 +515,7 @@ resource "aws_eks_addon" "this" { module.self_managed_node_group, ] - tags = var.tags + tags = merge(var.tags, try(each.value.tags, {})) } resource "aws_eks_addon" "before_compute" { @@ -530,15 +538,7 @@ resource "aws_eks_addon" "before_compute" { delete = try(each.value.timeouts.delete, var.cluster_addons_timeouts.delete, null) } - tags = var.tags -} - -data "aws_eks_addon_version" "this" { - for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } - - addon_name = try(each.value.name, each.key) - kubernetes_version = coalesce(var.cluster_version, aws_eks_cluster.this[0].version) - most_recent = try(each.value.most_recent, null) + tags = merge(var.tags, try(each.value.tags, {})) } ################################################################################ @@ -562,5 +562,5 @@ resource "aws_eks_identity_provider_config" "this" { username_prefix = lookup(each.value, "username_prefix", null) } - tags = var.tags + tags = merge(var.tags, try(each.value.tags, {})) } From 6a1e124e99706ea29cb18d153d0061d8202546ea Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 21 Feb 2024 22:34:23 +0000 Subject: [PATCH 744/962] chore(release): version 20.3.0 [skip ci] ## [20.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.2.2...v20.3.0) (2024-02-21) ### Features * Add support for addon and identity provider custom tags ([#2938](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2938)) ([f6255c4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f6255c49e47d44bd62bb2b4e1e448ac80ceb2b3a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94178d1342..735cf7d7ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.2.2...v20.3.0) (2024-02-21) + + +### Features + +* Add support for addon and identity provider custom tags ([#2938](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2938)) ([f6255c4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f6255c49e47d44bd62bb2b4e1e448ac80ceb2b3a)) + ### [20.2.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.2.1...v20.2.2) (2024-02-21) From 7f472ec660049d4ca85de039cb3015c1b1d12fb8 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 23 Feb 2024 09:29:10 -0500 Subject: [PATCH 745/962] feat: Add support for enabling EFA resources (#2936) * feat: Add support for enabling EFA resources * feat: Add support for creating placement group and ensuring subnet ID used supports the instance type provided * chore: Update README and examples * feat: Update AWS provider MSV to support `maximum_network_cards` attribute * fix: Update self-managed example after last round of testing; improve EFA support wording --- .pre-commit-config.yaml | 2 +- README.md | 59 +++++++++++- examples/eks_managed_node_group/README.md | 4 +- examples/eks_managed_node_group/main.tf | 26 +++++- examples/eks_managed_node_group/versions.tf | 2 +- examples/fargate_profile/README.md | 4 +- examples/fargate_profile/versions.tf | 2 +- examples/karpenter/README.md | 6 +- examples/karpenter/versions.tf | 2 +- examples/outposts/README.md | 4 +- examples/outposts/prerequisites/versions.tf | 2 +- examples/outposts/versions.tf | 2 +- examples/self_managed_node_group/README.md | 4 +- examples/self_managed_node_group/main.tf | 25 +++++ examples/self_managed_node_group/versions.tf | 2 +- modules/eks-managed-node-group/README.md | 9 +- modules/eks-managed-node-group/main.tf | 96 ++++++++++++++++++-- modules/eks-managed-node-group/variables.tf | 6 ++ modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 +- modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 4 +- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 9 +- modules/self-managed-node-group/main.tf | 96 ++++++++++++++++++-- modules/self-managed-node-group/variables.tf | 6 ++ modules/self-managed-node-group/versions.tf | 2 +- node_groups.tf | 24 +++++ variables.tf | 6 ++ versions.tf | 2 +- 30 files changed, 366 insertions(+), 50 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9af61e655b..0c542c8442 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.87.1 + rev: v1.88.0 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/README.md b/README.md index 7c6c65a421..1b763ec7de 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,60 @@ On clusters that were created prior to CAM support, there will be an existing ac Setting the `bootstrap_cluster_creator_admin_permissions` is a one time operation when the cluster is created; it cannot be modified later through the EKS API. In this project we are hardcoding this to `false`. If users wish to achieve the same functionality, we will do that through an access entry which can be enabled or disabled at any time of their choosing using the variable `enable_cluster_creator_admin_permissions` +### Enabling EFA Support + +When enabling EFA support via `enable_efa_support = true`, there are two locations this can be specified - one at the cluster level, and one at the nodegroup level. Enabling at the cluster level will add the EFA required ingress/egress rules to the shared security group created for the nodegroup(s). Enabling at the nodegroup level will do the following (per nodegroup where enabled): + +1. All EFA interfaces supported by the instance will be exposed on the launch template used by the nodegroup +2. A placement group with `strategy = "clustered"` per EFA requirements is created and passed to the launch template used by the nodegroup +3. Data sources will reverse lookup the availability zones that support the instance type selected based on the subnets provided, ensuring that only the associated subnets are passed to the launch template and therefore used by the placement group. This avoids the placement group being created in an availability zone that does not support the instance type selected. + +> [!TIP] +> Use the [aws-efa-k8s-device-plugin](https://github.com/aws/eks-charts/tree/master/stable/aws-efa-k8s-device-plugin) Helm chart to expose the EFA interfaces on the nodes as an extended resource, and allow pods to request the interfaces be mounted to their containers. +> +> The EKS AL2 GPU AMI comes with the necessary EFA components pre-installed - you just need to expose the EFA devices on the nodes via their launch templates, ensure the required EFA security group rules are in place, and deploy the `aws-efa-k8s-device-plugin` in order to start utilizing EFA within your cluster. Your application container will need to have the necessary libraries and runtime in order to utilize communication over the EFA interfaces (NCCL, aws-ofi-nccl, hwloc, libfabric, aws-neuornx-collectives, CUDA, etc.). + +If you disable the creation and use of the managed nodegroup custom launch template (`create_launch_template = false` and/or `use_custom_launch_template = false`), this will interfere with the EFA functionality provided. In addition, if you do not supply an `instance_type` for self-managed nodegroup(s), or `instance_types` for the managed nodegroup(s), this will also interfere with the functionality. In order to support the EFA functionality provided by `enable_efa_support = true`, you must utilize the custom launch template created/provided by this module, and supply an `instance_type`/`instance_types` for the respective nodegroup. + +The logic behind supporting EFA uses a data source to lookup the instance type to retrieve the number of interfaces that the instance supports in order to enumerate and expose those interfaces on the launch template created. For managed nodegroups where a list of instance types are supported, the first instance type in the list is used to calculate the number of EFA interfaces supported. Mixing instance types with varying number of interfaces is not recommended for EFA (or in some cases, mixing instance types is not supported - i.e. - p5.48xlarge and p4d.24xlarge). In addition to exposing the EFA interfaces and updating the security group rules, a placement group is created per the EFA requirements and only the availability zones that support the instance type selected are used in the subnets provided to the nodegroup. + +In order to enable EFA support, you will have to specify `enable_efa_support = true` on both the cluster and each nodegroup that you wish to enable EFA support for: + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + # Truncated for brevity ... + + # Adds the EFA required security group rules to the shared + # security group created for the nodegroup(s) + enable_efa_support = true + + eks_managed_node_groups = { + example = { + instance_types = ["p5.48xlarge"] + + # Exposes all EFA interfaces on the launch template created by the nodegroup(s) + # This would expose all 32 EFA interfaces for the p5.48xlarge instance type + enable_efa_support = true + + pre_bootstrap_user_data = <<-EOT + # Mount NVME instance store volumes since they are typically + # available on instance types that support EFA + setup-local-disks raid0 + EOT + + # EFA should only be enabled when connecting 2 or more nodes + # Do not use EFA on a single node workload + min_size = 2 + max_size = 10 + desired_size = 2 + } + } +} +``` + ## Examples - [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group): EKS Cluster using EKS managed node groups @@ -135,7 +189,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -143,7 +197,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | @@ -240,6 +294,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | | [enable\_cluster\_creator\_admin\_permissions](#input\_enable\_cluster\_creator\_admin\_permissions) | Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry | `bool` | `false` | no | +| [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | | [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled | `bool` | `true` | no | | [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no | diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index e94558cd77..1772d1324a 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -30,13 +30,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index de737d70fb..370bb5755a 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -7,7 +7,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.27" + cluster_version = "1.29" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -37,6 +37,10 @@ module "eks" { enable_cluster_creator_admin_permissions = true + # Enable EFA support by adding necessary security group rules + # to the shared node security group + enable_efa_support = true + cluster_addons = { coredns = { most_recent = true @@ -241,6 +245,26 @@ module "eks" { ExtraTag = "EKS managed node group complete example" } } + + efa = { + # Disabling automatic creation due to instance type/quota availability + # Can be enabled when appropriate for testing/validation + create = false + + instance_types = ["trn1n.32xlarge"] + ami_type = "AL2_x86_64_GPU" + + enable_efa_support = true + pre_bootstrap_user_data = <<-EOT + # Mount NVME instance store volumes since they are typically + # available on instances that support EFA + setup-local-disks raid0 + EOT + + min_size = 2 + max_size = 2 + desired_size = 2 + } } access_entries = { diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 9ea3bf1871..55e303ce15 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -20,13 +20,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index a1303999b0..f36f22c2e5 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -55,7 +55,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -63,8 +63,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.38 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 96d7b4ea19..711da76a33 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } helm = { source = "hashicorp/helm" diff --git a/examples/outposts/README.md b/examples/outposts/README.md index 30fffaf330..87bdafa73a 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -49,14 +49,14 @@ terraform destroy | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | | [kubernetes](#requirement\_kubernetes) | >= 2.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | | [kubernetes](#provider\_kubernetes) | >= 2.20 | ## Modules diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/examples/outposts/prerequisites/versions.tf +++ b/examples/outposts/prerequisites/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf index dff26f6939..73a045bb35 100644 --- a/examples/outposts/versions.tf +++ b/examples/outposts/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index c19999d9b9..2566c3d9cb 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -26,13 +26,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index dc125e1fbb..433e644993 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -31,6 +31,12 @@ module "eks" { cluster_version = local.cluster_version cluster_endpoint_public_access = true + enable_cluster_creator_admin_permissions = true + + # Enable EFA support by adding necessary security group rules + # to the shared node security group + enable_efa_support = true + cluster_addons = { coredns = { most_recent = true @@ -252,6 +258,25 @@ module "eks" { ExtraTag = "Self managed node group complete example" } } + + efa = { + # Disabling automatic creation due to instance type/quota availability + # Can be enabled when appropriate for testing/validation + create = false + + instance_type = "trn1n.32xlarge" + + enable_efa_support = true + pre_bootstrap_user_data = <<-EOT + # Mount NVME instance store volumes since they are typically + # available on instances that support EFA + setup-local-disks raid0 + EOT + + min_size = 2 + max_size = 2 + desired_size = 2 + } } tags = local.tags diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index ebae013b92..708f7ad1f6 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules @@ -88,9 +88,13 @@ module "eks_managed_node_group" { | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_placement_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/placement_group) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | +| [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_subnets.efa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs @@ -123,6 +127,7 @@ module "eks_managed_node_group" { | [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | | [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | | [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template. Only valid when using a custom AMI via `ami_id` | `bool` | `false` | no | +| [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | | [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | | [force\_update\_version](#input\_force\_update\_version) | Force version update if existing pods are unable to be drained due to a pod disruption budget issue | `bool` | `null` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 16ca010ae6..fa74cd3b76 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -24,6 +24,33 @@ module "user_data" { user_data_template_path = var.user_data_template_path } +################################################################################ +# EFA Support +################################################################################ + +data "aws_ec2_instance_type" "this" { + count = var.enable_efa_support ? 1 : 0 + + instance_type = local.efa_instance_type +} + +locals { + efa_instance_type = try(element(var.instance_types, 0), "") + num_network_cards = try(data.aws_ec2_instance_type.this[0].maximum_network_cards, 0) + + efa_network_interfaces = [ + for i in range(local.num_network_cards) : { + associate_public_ip_address = false + delete_on_termination = true + device_index = i == 0 ? 0 : 1 + network_card_index = i + interface_type = "efa" + } + ] + + network_interfaces = var.enable_efa_support ? local.efa_network_interfaces : var.network_interfaces +} + ################################################################################ # Launch template ################################################################################ @@ -31,6 +58,8 @@ module "user_data" { locals { launch_template_name = coalesce(var.launch_template_name, "${var.name}-eks-node-group") security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) + + placement = var.create && var.enable_efa_support ? { group_name = aws_placement_group.this[0].name } : var.placement } resource "aws_launch_template" "this" { @@ -215,7 +244,8 @@ resource "aws_launch_template" "this" { name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null dynamic "network_interfaces" { - for_each = var.network_interfaces + for_each = local.network_interfaces + content { associate_carrier_ip_address = try(network_interfaces.value.associate_carrier_ip_address, null) associate_public_ip_address = try(network_interfaces.value.associate_public_ip_address, null) @@ -243,14 +273,14 @@ resource "aws_launch_template" "this" { } dynamic "placement" { - for_each = length(var.placement) > 0 ? [var.placement] : [] + for_each = length(local.placement) > 0 ? [local.placement] : [] content { affinity = try(placement.value.affinity, null) - availability_zone = try(placement.value.availability_zone, null) - group_name = try(placement.value.group_name, null) - host_id = try(placement.value.host_id, null) - host_resource_group_arn = try(placement.value.host_resource_group_arn, null) + availability_zone = lookup(placement.value, "availability_zone", null) + group_name = lookup(placement.value, "group_name", null) + host_id = lookup(placement.value, "host_id", null) + host_resource_group_arn = lookup(placement.value, "host_resource_group_arn", null) partition_number = try(placement.value.partition_number, null) spread_domain = try(placement.value.spread_domain, null) tenancy = try(placement.value.tenancy, null) @@ -280,7 +310,7 @@ resource "aws_launch_template" "this" { update_default_version = var.update_launch_template_default_version user_data = module.user_data.user_data - vpc_security_group_ids = length(var.network_interfaces) > 0 ? [] : local.security_group_ids + vpc_security_group_ids = length(local.network_interfaces) > 0 ? [] : local.security_group_ids tags = var.tags @@ -311,7 +341,7 @@ resource "aws_eks_node_group" "this" { # Required cluster_name = var.cluster_name node_role_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn - subnet_ids = var.subnet_ids + subnet_ids = var.enable_efa_support ? data.aws_subnets.efa[0].ids : var.subnet_ids scaling_config { min_size = var.min_size @@ -448,6 +478,56 @@ resource "aws_iam_role_policy_attachment" "additional" { role = aws_iam_role.this[0].name } +################################################################################ +# Placement Group +################################################################################ + +resource "aws_placement_group" "this" { + count = var.create && var.enable_efa_support ? 1 : 0 + + name = "${var.cluster_name}-${var.name}" + strategy = "cluster" + + tags = var.tags +} + +################################################################################ +# Instance AZ Lookup + +# Instances usually used in placement groups w/ EFA are only available in +# select availability zones. These data sources will cross reference the availability +# zones supported by the instance type with the subnets provided to ensure only +# AZs/subnets that are supported are used. +################################################################################ + +# Find the availability zones supported by the instance type +data "aws_ec2_instance_type_offerings" "this" { + count = var.create && var.enable_efa_support ? 1 : 0 + + filter { + name = "instance-type" + values = [local.efa_instance_type] + } + + location_type = "availability-zone-id" +} + +# Reverse the lookup to find one of the subnets provided based on the availability +# availability zone ID of the queried instance type (supported) +data "aws_subnets" "efa" { + count = var.create && var.enable_efa_support ? 1 : 0 + + filter { + name = "subnet-id" + values = var.subnet_ids + } + + filter { + name = "availability-zone-id" + values = data.aws_ec2_instance_type_offerings.this[0].locations + } +} + ################################################################################ # Autoscaling Group Schedule ################################################################################ diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index ede9dc4c50..192980888a 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -250,6 +250,12 @@ variable "enable_monitoring" { default = true } +variable "enable_efa_support" { + description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" + type = bool + default = false +} + variable "network_interfaces" { description = "Customize network interfaces to be attached at instance boot time" type = list(any) diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 8656a6f191..1fb59c4ba0 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 00f724acc5..d6f27bfd1f 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -85,13 +85,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 83e44d0bc5..6fe6e43180 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules @@ -69,10 +69,14 @@ module "self_managed_node_group" { | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_placement_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/placement_group) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | +| [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_subnets.efa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs @@ -108,6 +112,7 @@ module "self_managed_node_group" { | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance will be EBS-optimized | `bool` | `null` | no | | [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | | [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | +| [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | | [enabled\_metrics](#input\_enabled\_metrics) | A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances` | `list(string)` | `[]` | no | | [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index eb3192970b..bb1eea0b54 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -35,6 +35,33 @@ module "user_data" { user_data_template_path = var.user_data_template_path } +################################################################################ +# EFA Support +################################################################################ + +data "aws_ec2_instance_type" "this" { + count = var.enable_efa_support && local.instance_type_provided ? 1 : 0 + + instance_type = var.instance_type +} + +locals { + instance_type_provided = var.instance_type != "" + num_network_cards = try(data.aws_ec2_instance_type.this[0].maximum_network_cards, 0) + + efa_network_interfaces = [ + for i in range(local.num_network_cards) : { + associate_public_ip_address = false + delete_on_termination = true + device_index = i == 0 ? 0 : 1 + network_card_index = i + interface_type = "efa" + } + ] + + network_interfaces = var.enable_efa_support && local.instance_type_provided ? local.efa_network_interfaces : var.network_interfaces +} + ################################################################################ # Launch template ################################################################################ @@ -42,6 +69,8 @@ module "user_data" { locals { launch_template_name = coalesce(var.launch_template_name, "${var.name}-node-group") security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) + + placement = var.create && var.enable_efa_support ? { group_name = aws_placement_group.this[0].name } : var.placement } resource "aws_launch_template" "this" { @@ -321,7 +350,8 @@ resource "aws_launch_template" "this" { name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null dynamic "network_interfaces" { - for_each = var.network_interfaces + for_each = local.network_interfaces + content { associate_carrier_ip_address = try(network_interfaces.value.associate_carrier_ip_address, null) associate_public_ip_address = try(network_interfaces.value.associate_public_ip_address, null) @@ -347,14 +377,14 @@ resource "aws_launch_template" "this" { } dynamic "placement" { - for_each = length(var.placement) > 0 ? [var.placement] : [] + for_each = length(local.placement) > 0 ? [local.placement] : [] content { affinity = try(placement.value.affinity, null) - availability_zone = try(placement.value.availability_zone, null) - group_name = try(placement.value.group_name, null) - host_id = try(placement.value.host_id, null) - host_resource_group_arn = try(placement.value.host_resource_group_arn, null) + availability_zone = lookup(placement.value, "availability_zone", null) + group_name = lookup(placement.value, "group_name", null) + host_id = lookup(placement.value, "host_id", null) + host_resource_group_arn = lookup(placement.value, "host_resource_group_arn", null) partition_number = try(placement.value.partition_number, null) spread_domain = try(placement.value.spread_domain, null) tenancy = try(placement.value.tenancy, null) @@ -384,7 +414,7 @@ resource "aws_launch_template" "this" { update_default_version = var.update_launch_template_default_version user_data = module.user_data.user_data - vpc_security_group_ids = length(var.network_interfaces) > 0 ? [] : local.security_group_ids + vpc_security_group_ids = length(local.network_interfaces) > 0 ? [] : local.security_group_ids tags = var.tags @@ -664,7 +694,7 @@ resource "aws_autoscaling_group" "this" { target_group_arns = var.target_group_arns termination_policies = var.termination_policies - vpc_zone_identifier = var.subnet_ids + vpc_zone_identifier = var.enable_efa_support ? data.aws_subnets.efa[0].ids : var.subnet_ids wait_for_capacity_timeout = var.wait_for_capacity_timeout wait_for_elb_capacity = var.wait_for_elb_capacity @@ -771,6 +801,56 @@ resource "aws_iam_instance_profile" "this" { } } +################################################################################ +# Placement Group +################################################################################ + +resource "aws_placement_group" "this" { + count = var.create && var.enable_efa_support ? 1 : 0 + + name = "${var.cluster_name}-${var.name}" + strategy = "cluster" + + tags = var.tags +} + +################################################################################ +# Instance AZ Lookup + +# Instances usually used in placement groups w/ EFA are only available in +# select availability zones. These data sources will cross reference the availability +# zones supported by the instance type with the subnets provided to ensure only +# AZs/subnets that are supported are used. +################################################################################ + +# Find the availability zones supported by the instance type +data "aws_ec2_instance_type_offerings" "this" { + count = var.create && var.enable_efa_support ? 1 : 0 + + filter { + name = "instance-type" + values = [var.instance_type] + } + + location_type = "availability-zone-id" +} + +# Reverse the lookup to find one of the subnets provided based on the availability +# availability zone ID of the queried instance type (supported) +data "aws_subnets" "efa" { + count = var.create && var.enable_efa_support ? 1 : 0 + + filter { + name = "subnet-id" + values = var.subnet_ids + } + + filter { + name = "availability-zone-id" + values = data.aws_ec2_instance_type_offerings.this[0].locations + } +} + ################################################################################ # Access Entry ################################################################################ diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index c6c5086a57..54bded55fe 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -270,6 +270,12 @@ variable "enable_monitoring" { default = true } +variable "enable_efa_support" { + description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" + type = bool + default = false +} + variable "metadata_options" { description = "Customize the metadata options for the instance" type = map(string) diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/node_groups.tf b/node_groups.tf index ddfc20fa09..a225bb8931 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -179,6 +179,27 @@ locals { ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null } } : k => v if var.node_security_group_enable_recommended_rules } + + efa_security_group_rules = { for k, v in + { + ingress_all_self_efa = { + description = "Node to node EFA" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + egress_all_self_efa = { + description = "Node to node EFA" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + self = true + } + } : k => v if var.enable_efa_support + } } resource "aws_security_group" "node" { @@ -205,6 +226,7 @@ resource "aws_security_group" "node" { resource "aws_security_group_rule" "node" { for_each = { for k, v in merge( + local.efa_security_group_rules, local.node_security_group_rules, local.node_security_group_recommended_rules, var.node_security_group_additional_rules, @@ -343,6 +365,7 @@ module "eks_managed_node_group" { license_specifications = try(each.value.license_specifications, var.eks_managed_node_group_defaults.license_specifications, {}) metadata_options = try(each.value.metadata_options, var.eks_managed_node_group_defaults.metadata_options, local.metadata_options) enable_monitoring = try(each.value.enable_monitoring, var.eks_managed_node_group_defaults.enable_monitoring, true) + enable_efa_support = try(each.value.enable_efa_support, var.eks_managed_node_group_defaults.enable_efa_support, false) network_interfaces = try(each.value.network_interfaces, var.eks_managed_node_group_defaults.network_interfaces, []) placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, {}) maintenance_options = try(each.value.maintenance_options, var.eks_managed_node_group_defaults.maintenance_options, {}) @@ -478,6 +501,7 @@ module "self_managed_node_group" { license_specifications = try(each.value.license_specifications, var.self_managed_node_group_defaults.license_specifications, {}) metadata_options = try(each.value.metadata_options, var.self_managed_node_group_defaults.metadata_options, local.metadata_options) enable_monitoring = try(each.value.enable_monitoring, var.self_managed_node_group_defaults.enable_monitoring, true) + enable_efa_support = try(each.value.enable_efa_support, var.self_managed_node_group_defaults.enable_efa_support, false) network_interfaces = try(each.value.network_interfaces, var.self_managed_node_group_defaults.network_interfaces, []) placement = try(each.value.placement, var.self_managed_node_group_defaults.placement, {}) maintenance_options = try(each.value.maintenance_options, var.self_managed_node_group_defaults.maintenance_options, {}) diff --git a/variables.tf b/variables.tf index 83776d6f4f..fff8895279 100644 --- a/variables.tf +++ b/variables.tf @@ -376,6 +376,12 @@ variable "node_security_group_tags" { default = {} } +variable "enable_efa_support" { + description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" + type = bool + default = false +} + ################################################################################ # IRSA ################################################################################ diff --git a/versions.tf b/versions.tf index a7084a7252..4d006cbfb6 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } tls = { source = "hashicorp/tls" From e6c3e90a0268b05e9a4c6f5e51bbbe593538a696 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 23 Feb 2024 14:29:35 +0000 Subject: [PATCH 746/962] chore(release): version 20.4.0 [skip ci] ## [20.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.3.0...v20.4.0) (2024-02-23) ### Features * Add support for enabling EFA resources ([#2936](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2936)) ([7f472ec](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7f472ec660049d4ca85de039cb3015c1b1d12fb8)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 735cf7d7ee..95042c5715 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.3.0...v20.4.0) (2024-02-23) + + +### Features + +* Add support for enabling EFA resources ([#2936](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2936)) ([7f472ec](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7f472ec660049d4ca85de039cb3015c1b1d12fb8)) + ## [20.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.2.2...v20.3.0) (2024-02-21) From 7c99bb19cdbf1eb4f4543f9b8e6d29c3a6734a55 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 29 Feb 2024 19:12:50 -0500 Subject: [PATCH 747/962] feat: Add support for AL2023 `nodeadm` user data (#2942) --- docs/UPGRADE-20.0.md | 1 + examples/eks_managed_node_group/main.tf | 23 ++ examples/self_managed_node_group/main.tf | 23 ++ examples/user_data/README.md | 14 ++ examples/user_data/main.tf | 198 +++++++++++++++++- examples/user_data/outputs.tf | 43 ++++ .../rendered/al2023/eks-mng-additional.sh | 19 ++ .../rendered/al2023/eks-mng-custom-ami.sh | 41 ++++ .../al2023/eks-mng-custom-template.sh | 44 ++++ .../rendered/al2023/eks-mng-no-op.sh | 0 .../rendered/al2023/self-mng-bootstrap.sh | 40 ++++ .../al2023/self-mng-custom-template.sh | 44 ++++ .../rendered/al2023/self-mng-no-op.sh | 0 .../user_data/templates/al2023_custom.tpl | 14 ++ modules/_user_data/README.md | 3 + modules/_user_data/main.tf | 95 +++++---- modules/_user_data/variables.tf | 22 ++ templates/al2023_user_data.tpl | 13 ++ 18 files changed, 599 insertions(+), 38 deletions(-) create mode 100755 examples/user_data/rendered/al2023/eks-mng-additional.sh create mode 100755 examples/user_data/rendered/al2023/eks-mng-custom-ami.sh create mode 100755 examples/user_data/rendered/al2023/eks-mng-custom-template.sh create mode 100755 examples/user_data/rendered/al2023/eks-mng-no-op.sh create mode 100755 examples/user_data/rendered/al2023/self-mng-bootstrap.sh create mode 100755 examples/user_data/rendered/al2023/self-mng-custom-template.sh create mode 100755 examples/user_data/rendered/al2023/self-mng-no-op.sh create mode 100644 examples/user_data/templates/al2023_custom.tpl create mode 100644 templates/al2023_user_data.tpl diff --git a/docs/UPGRADE-20.0.md b/docs/UPGRADE-20.0.md index 5ae1b35783..1f16712fda 100644 --- a/docs/UPGRADE-20.0.md +++ b/docs/UPGRADE-20.0.md @@ -20,6 +20,7 @@ To give users advanced notice and provide some future direction for this module, 1. The `aws-auth` sub-module will be removed entirely from the project. Since this sub-module is captured in the v20.x releases, users can continue using it even after the module moves forward with the next major version. The long term strategy and direction is cluster access entry and to rely only on the AWS Terraform provider. 2. The default value for `authentication_mode` will change to `API`. Aligning with point 1 above, this is a one way change, but users are free to specify the value of their choosing in place of this default (when the change is made). This module will proceed with an EKS API first strategy. 3. The launch template and autoscaling group usage contained within the EKS managed nodegroup and self-managed nodegroup sub-modules *might be replaced with the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module. At minimum, it makes sense to replace most of functionality in the self-managed nodegroup module with this external module, but its not yet clear if there is any benefit of using it in the EKS managed nodegroup sub-module. The interface that users interact with will stay the same, the changes will be internal to the implementation and we will do everything we can to keep the disruption to a minimum. +4. The `platform` variable will be replaced and instead `ami_type` will become the standard across both self-managed nodegroup(s) and EKS managed nodegroup(s). As EKS expands its portfolio of supported operating systems, the `ami_type` is better suited to associate the correct user data format to the respective OS. The `platform` variable is a legacy artifact of self-managed nodegroups but not as descriptive as the `ami_type`, and therefore it will be removed in favor of `ami_type`. ## Additional changes diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 370bb5755a..17ff2bbc80 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -86,6 +86,29 @@ module "eks" { } } + # AL2023 node group utilizing new user data format which utilizes nodeadm + # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh) + al2023_nodeadm = { + platform = "al2023" + + cloudinit_pre_nodeadm = [ + { + content_type = "application/node.eks.aws" + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + } + ] + } + # Default node group - as provided by AWS EKS using Bottlerocket bottlerocket_default = { # By default, the module creates a launch template to ensure tags are propagated to instances, etc., diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 433e644993..a9f228e117 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -72,6 +72,29 @@ module "eks" { # Default node group - as provisioned by the module defaults default_node_group = {} + # AL2023 node group utilizing new user data format which utilizes nodeadm + # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh) + al2023_nodeadm = { + platform = "al2023" + + cloudinit_pre_nodeadm = [ + { + content_type = "application/node.eks.aws" + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + } + ] + } + # Bottlerocket node group bottlerocket = { name = "bottlerocket-self-mng" diff --git a/examples/user_data/README.md b/examples/user_data/README.md index 6e1605ca1c..a84b75ac5e 100644 --- a/examples/user_data/README.md +++ b/examples/user_data/README.md @@ -30,6 +30,10 @@ $ terraform apply | Name | Source | Version | |------|--------|---------| +| [eks\_mng\_al2023\_additional](#module\_eks\_mng\_al2023\_additional) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2023\_custom\_ami](#module\_eks\_mng\_al2023\_custom\_ami) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2023\_custom\_template](#module\_eks\_mng\_al2023\_custom\_template) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2023\_no\_op](#module\_eks\_mng\_al2023\_no\_op) | ../../modules/_user_data | n/a | | [eks\_mng\_al2\_additional](#module\_eks\_mng\_al2\_additional) | ../../modules/_user_data | n/a | | [eks\_mng\_al2\_custom\_ami](#module\_eks\_mng\_al2\_custom\_ami) | ../../modules/_user_data | n/a | | [eks\_mng\_al2\_custom\_template](#module\_eks\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a | @@ -42,6 +46,9 @@ $ terraform apply | [eks\_mng\_windows\_custom\_ami](#module\_eks\_mng\_windows\_custom\_ami) | ../../modules/_user_data | n/a | | [eks\_mng\_windows\_custom\_template](#module\_eks\_mng\_windows\_custom\_template) | ../../modules/_user_data | n/a | | [eks\_mng\_windows\_no\_op](#module\_eks\_mng\_windows\_no\_op) | ../../modules/_user_data | n/a | +| [self\_mng\_al2023\_bootstrap](#module\_self\_mng\_al2023\_bootstrap) | ../../modules/_user_data | n/a | +| [self\_mng\_al2023\_custom\_template](#module\_self\_mng\_al2023\_custom\_template) | ../../modules/_user_data | n/a | +| [self\_mng\_al2023\_no\_op](#module\_self\_mng\_al2023\_no\_op) | ../../modules/_user_data | n/a | | [self\_mng\_al2\_bootstrap](#module\_self\_mng\_al2\_bootstrap) | ../../modules/_user_data | n/a | | [self\_mng\_al2\_custom\_template](#module\_self\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a | | [self\_mng\_al2\_no\_op](#module\_self\_mng\_al2\_no\_op) | ../../modules/_user_data | n/a | @@ -56,6 +63,10 @@ $ terraform apply | Name | Type | |------|------| +| [local_file.eks_mng_al2023_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_al2023_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_al2023_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_al2023_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_al2_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_al2_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | @@ -68,6 +79,9 @@ $ terraform apply | [local_file.eks_mng_windows_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_windows_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_windows_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_al2023_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_al2023_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_al2023_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_al2_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_al2_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index 58536357d5..6a90081ec0 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -43,7 +43,6 @@ module "eks_mng_al2_custom_ami" { EOT } - module "eks_mng_al2_custom_template" { source = "../../modules/_user_data" @@ -65,6 +64,107 @@ module "eks_mng_al2_custom_template" { EOT } +################################################################################ +# EKS managed node group - AL2023 +################################################################################ + +module "eks_mng_al2023_no_op" { + source = "../../modules/_user_data" + + platform = "al2023" +} + +module "eks_mng_al2023_additional" { + source = "../../modules/_user_data" + + platform = "al2023" + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] +} + +module "eks_mng_al2023_custom_ami" { + source = "../../modules/_user_data" + + platform = "al2023" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + + enable_bootstrap_user_data = true + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] + + cloudinit_post_nodeadm = [{ + content = <<-EOT + echo "All done" + EOT + content_type = "text/x-shellscript; charset=\"us-ascii\"" + }] +} + +module "eks_mng_al2023_custom_template" { + source = "../../modules/_user_data" + + platform = "al2023" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + enable_bootstrap_user_data = true + user_data_template_path = "${path.module}/templates/al2023_custom.tpl" + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] + + cloudinit_post_nodeadm = [{ + content = <<-EOT + echo "All done" + EOT + content_type = "text/x-shellscript; charset=\"us-ascii\"" + }] +} + ################################################################################ # EKS managed node group - Bottlerocket ################################################################################ @@ -80,6 +180,9 @@ module "eks_mng_bottlerocket_additional" { platform = "bottlerocket" + # Should do nothing + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + bootstrap_extra_args = <<-EOT # extra args added [settings.kernel] @@ -138,6 +241,9 @@ module "eks_mng_windows_additional" { platform = "windows" + # Should do nothing + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + pre_bootstrap_user_data = <<-EOT [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' EOT @@ -245,6 +351,90 @@ module "self_mng_al2_custom_template" { EOT } +################################################################################ +# Self-managed node group - AL2023 +################################################################################ + +module "self_mng_al2023_no_op" { + source = "../../modules/_user_data" + + platform = "al2023" + + is_eks_managed_node_group = false +} + +module "self_mng_al2023_bootstrap" { + source = "../../modules/_user_data" + + platform = "al2023" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] + + cloudinit_post_nodeadm = [{ + content = <<-EOT + echo "All done" + EOT + content_type = "text/x-shellscript; charset=\"us-ascii\"" + }] +} + +module "self_mng_al2023_custom_template" { + source = "../../modules/_user_data" + + platform = "al2023" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + user_data_template_path = "${path.module}/templates/al2023_custom.tpl" + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] + + cloudinit_post_nodeadm = [{ + content = <<-EOT + echo "All done" + EOT + content_type = "text/x-shellscript; charset=\"us-ascii\"" + }] +} + ################################################################################ # Self-managed node group - Bottlerocket ################################################################################ @@ -269,6 +459,9 @@ module "self_mng_bottlerocket_bootstrap" { cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 + # Should do nothing + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + bootstrap_extra_args = <<-EOT # extra args added [settings.kernel] @@ -321,6 +514,9 @@ module "self_mng_windows_bootstrap" { cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 + # Should do nothing + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + pre_bootstrap_user_data = <<-EOT [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' EOT diff --git a/examples/user_data/outputs.tf b/examples/user_data/outputs.tf index 22988e9d43..b407ccec2b 100644 --- a/examples/user_data/outputs.tf +++ b/examples/user_data/outputs.tf @@ -30,6 +30,30 @@ resource "local_file" "eks_mng_al2_custom_template" { filename = "${path.module}/rendered/al2/eks-mng-custom-template.sh" } +################################################################################ +# EKS managed node group - AL2023 +################################################################################ + +resource "local_file" "eks_mng_al2023_no_op" { + content = base64decode(module.eks_mng_al2023_no_op.user_data) + filename = "${path.module}/rendered/al2023/eks-mng-no-op.sh" +} + +resource "local_file" "eks_mng_al2023_additional" { + content = base64decode(module.eks_mng_al2023_additional.user_data) + filename = "${path.module}/rendered/al2023/eks-mng-additional.sh" +} + +resource "local_file" "eks_mng_al2023_custom_ami" { + content = base64decode(module.eks_mng_al2023_custom_ami.user_data) + filename = "${path.module}/rendered/al2023/eks-mng-custom-ami.sh" +} + +resource "local_file" "eks_mng_al2023_custom_template" { + content = base64decode(module.eks_mng_al2023_custom_template.user_data) + filename = "${path.module}/rendered/al2023/eks-mng-custom-template.sh" +} + ################################################################################ # EKS managed node group - Bottlerocket ################################################################################ @@ -97,6 +121,25 @@ resource "local_file" "self_mng_al2_custom_template" { filename = "${path.module}/rendered/al2/self-mng-custom-template.sh" } +################################################################################ +# Self-managed node group - AL2023 +################################################################################ + +resource "local_file" "self_mng_al2023_no_op" { + content = base64decode(module.self_mng_al2023_no_op.user_data) + filename = "${path.module}/rendered/al2023/self-mng-no-op.sh" +} + +resource "local_file" "self_mng_al2023_bootstrap" { + content = base64decode(module.self_mng_al2023_bootstrap.user_data) + filename = "${path.module}/rendered/al2023/self-mng-bootstrap.sh" +} + +resource "local_file" "self_mng_al2023_custom_template" { + content = base64decode(module.self_mng_al2023_custom_template.user_data) + filename = "${path.module}/rendered/al2023/self-mng-custom-template.sh" +} + ################################################################################ # Self-managed node group - Bottlerocket ################################################################################ diff --git a/examples/user_data/rendered/al2023/eks-mng-additional.sh b/examples/user_data/rendered/al2023/eks-mng-additional.sh new file mode 100755 index 0000000000..d8d3a1fdb4 --- /dev/null +++ b/examples/user_data/rendered/al2023/eks-mng-additional.sh @@ -0,0 +1,19 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha +kind: NodeConfig +spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh b/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh new file mode 100755 index 0000000000..fe8f07a24b --- /dev/null +++ b/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh @@ -0,0 +1,41 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha +kind: NodeConfig +spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ex-user-data + apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com + certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + cidr: 172.16.0.0/16 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + +echo "All done" + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/eks-mng-custom-template.sh b/examples/user_data/rendered/al2023/eks-mng-custom-template.sh new file mode 100755 index 0000000000..6267b85f72 --- /dev/null +++ b/examples/user_data/rendered/al2023/eks-mng-custom-template.sh @@ -0,0 +1,44 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha +kind: NodeConfig +spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ex-user-data + apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com + certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + containerd: + config: | + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + +echo "All done" + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/eks-mng-no-op.sh b/examples/user_data/rendered/al2023/eks-mng-no-op.sh new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/rendered/al2023/self-mng-bootstrap.sh b/examples/user_data/rendered/al2023/self-mng-bootstrap.sh new file mode 100755 index 0000000000..9ea008476e --- /dev/null +++ b/examples/user_data/rendered/al2023/self-mng-bootstrap.sh @@ -0,0 +1,40 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha +kind: NodeConfig +spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ex-user-data + apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com + certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + +echo "All done" + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/self-mng-custom-template.sh b/examples/user_data/rendered/al2023/self-mng-custom-template.sh new file mode 100755 index 0000000000..6267b85f72 --- /dev/null +++ b/examples/user_data/rendered/al2023/self-mng-custom-template.sh @@ -0,0 +1,44 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha +kind: NodeConfig +spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ex-user-data + apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com + certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + containerd: + config: | + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + +echo "All done" + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/self-mng-no-op.sh b/examples/user_data/rendered/al2023/self-mng-no-op.sh new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/templates/al2023_custom.tpl b/examples/user_data/templates/al2023_custom.tpl new file mode 100644 index 0000000000..a33aa44bf2 --- /dev/null +++ b/examples/user_data/templates/al2023_custom.tpl @@ -0,0 +1,14 @@ +%{ if enable_bootstrap_user_data ~} +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ${cluster_name} + apiServerEndpoint: ${cluster_endpoint} + certificateAuthority: ${cluster_auth_base64} + containerd: + config: | + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false +%{ endif ~} diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 4b50bfe35b..89edf259a5 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -26,6 +26,7 @@ No modules. | Name | Type | |------|------| +| [cloudinit_config.al2023_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | | [cloudinit_config.linux_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | ## Inputs @@ -33,6 +34,8 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index 5d9e669538..b70279bd31 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -1,65 +1,49 @@ locals { - int_linux_default_user_data = var.create && var.platform == "linux" && (var.enable_bootstrap_user_data || var.user_data_template_path != "") ? base64encode(templatefile( - coalesce(var.user_data_template_path, "${path.module}/../../templates/linux_user_data.tpl"), + template_path = { + al2023 = "${path.module}/../../templates/al2023_user_data.tpl" + bottlerocket = "${path.module}/../../templates/bottlerocket_user_data.tpl" + linux = "${path.module}/../../templates/linux_user_data.tpl" + windows = "${path.module}/../../templates/windows_user_data.tpl" + } + + user_data = base64encode(templatefile( + coalesce(var.user_data_template_path, local.template_path[var.platform]), { # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami enable_bootstrap_user_data = var.enable_bootstrap_user_data + # Required to bootstrap node cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 + # Optional cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr != null ? var.cluster_service_ipv4_cidr : "" bootstrap_extra_args = var.bootstrap_extra_args pre_bootstrap_user_data = var.pre_bootstrap_user_data post_bootstrap_user_data = var.post_bootstrap_user_data } - )) : "" + )) + platform = { + al2023 = { + user_data = var.create ? try(data.cloudinit_config.al2023_eks_managed_node_group[0].rendered, local.user_data) : "" + } bottlerocket = { - user_data = var.create && var.platform == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? base64encode(templatefile( - coalesce(var.user_data_template_path, "${path.module}/../../templates/bottlerocket_user_data.tpl"), - { - # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami - enable_bootstrap_user_data = var.enable_bootstrap_user_data - # Required to bootstrap node - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint - cluster_auth_base64 = var.cluster_auth_base64 - # Optional - is appended if using EKS managed node group without custom AMI - # cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr # Bottlerocket pulls this automatically https://github.com/bottlerocket-os/bottlerocket/issues/1866 - bootstrap_extra_args = var.bootstrap_extra_args - } - )) : "" + user_data = var.create && var.platform == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? local.user_data : "" } linux = { - user_data = try(data.cloudinit_config.linux_eks_managed_node_group[0].rendered, local.int_linux_default_user_data) - + user_data = var.create ? try(data.cloudinit_config.linux_eks_managed_node_group[0].rendered, local.user_data) : "" } windows = { - user_data = var.create && var.platform == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? base64encode(templatefile( - coalesce(var.user_data_template_path, "${path.module}/../../templates/windows_user_data.tpl"), - { - # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami - enable_bootstrap_user_data = var.enable_bootstrap_user_data - # Required to bootstrap node - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint - cluster_auth_base64 = var.cluster_auth_base64 - # Optional - is appended if using EKS managed node group without custom AMI - # cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr # Not supported yet: https://github.com/awslabs/amazon-eks-ami/issues/805 - bootstrap_extra_args = var.bootstrap_extra_args - pre_bootstrap_user_data = var.pre_bootstrap_user_data - post_bootstrap_user_data = var.post_bootstrap_user_data - } - )) : "" + user_data = var.create && var.platform == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? local.user_data : "" } } } # https://github.com/aws/containers-roadmap/issues/596#issuecomment-675097667 -# An important note is that user data must in MIME multi-part archive format, +# Managed nodegroup data must in MIME multi-part archive format, # as by default, EKS will merge the bootstrapping command required for nodes to join the # cluster with your user data. If you use a custom AMI in your launch template, # this merging will NOT happen and you are responsible for nodes joining the cluster. @@ -74,7 +58,44 @@ data "cloudinit_config" "linux_eks_managed_node_group" { # Prepend to existing user data supplied by AWS EKS part { - content_type = "text/x-shellscript" content = var.pre_bootstrap_user_data + content_type = "text/x-shellscript" + } +} + +# Scenarios: +# +# 1. Do nothing - provide nothing +# 2. Prepend stuff on EKS MNG (before EKS MNG adds its bit at the end) +# 3. Own all of the stuff on self-MNG or EKS MNG w/ custom AMI + +locals { + nodeadm_cloudinit = var.enable_bootstrap_user_data ? concat( + var.cloudinit_pre_nodeadm, + [{ + content_type = "application/node.eks.aws" + content = base64decode(local.user_data) + }], + var.cloudinit_post_nodeadm + ) : var.cloudinit_pre_nodeadm +} + +data "cloudinit_config" "al2023_eks_managed_node_group" { + count = var.create && var.platform == "al2023" && length(local.nodeadm_cloudinit) > 0 ? 1 : 0 + + base64_encode = true + gzip = false + boundary = "MIMEBOUNDARY" + + dynamic "part" { + # Using the index is fine in this context since any change in user data will be a replacement + for_each = { for i, v in local.nodeadm_cloudinit : i => v } + + content { + content = part.value.content + content_type = try(part.value.content_type, null) + filename = try(part.value.filename, null) + merge_type = try(part.value.merge_type, null) + } } } diff --git a/modules/_user_data/variables.tf b/modules/_user_data/variables.tf index 232e1e883e..96c1b07059 100644 --- a/modules/_user_data/variables.tf +++ b/modules/_user_data/variables.tf @@ -69,3 +69,25 @@ variable "user_data_template_path" { type = string default = "" } + +variable "cloudinit_pre_nodeadm" { + description = "Array of cloud-init document parts that are created before the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + +variable "cloudinit_post_nodeadm" { + description = "Array of cloud-init document parts that are created after the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} diff --git a/templates/al2023_user_data.tpl b/templates/al2023_user_data.tpl new file mode 100644 index 0000000000..820223f719 --- /dev/null +++ b/templates/al2023_user_data.tpl @@ -0,0 +1,13 @@ +%{ if enable_bootstrap_user_data ~} +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ${cluster_name} + apiServerEndpoint: ${cluster_endpoint} + certificateAuthority: ${cluster_auth_base64} +%{ if length(cluster_service_ipv4_cidr) > 0 ~} + cidr: ${cluster_service_ipv4_cidr} +%{ endif ~} +%{ endif ~} From 19b79df055512a7d61641fc716c9a2c597c7d277 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 1 Mar 2024 00:13:15 +0000 Subject: [PATCH 748/962] chore(release): version 20.5.0 [skip ci] ## [20.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.4.0...v20.5.0) (2024-03-01) ### Features * Add support for AL2023 `nodeadm` user data ([#2942](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2942)) ([7c99bb1](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7c99bb19cdbf1eb4f4543f9b8e6d29c3a6734a55)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95042c5715..aae04abad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.4.0...v20.5.0) (2024-03-01) + + +### Features + +* Add support for AL2023 `nodeadm` user data ([#2942](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2942)) ([7c99bb1](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7c99bb19cdbf1eb4f4543f9b8e6d29c3a6734a55)) + ## [20.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.3.0...v20.4.0) (2024-02-23) From d14cc925c450451b023407d05a2516d7682d1617 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 6 Mar 2024 21:30:34 -0500 Subject: [PATCH 749/962] fix: Update CI workflow versions to remove deprecated runtime warnings (#2956) --- .github/workflows/lock.yml | 2 +- .github/workflows/pr-title.yml | 2 +- .github/workflows/pre-commit.yml | 36 ++++++++++++++++++++-------- .github/workflows/release.yml | 12 +++++----- .github/workflows/stale-actions.yaml | 2 +- .pre-commit-config.yaml | 3 ++- 6 files changed, 37 insertions(+), 20 deletions(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 6b6c9cec02..bd5f2df7cb 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -8,7 +8,7 @@ jobs: lock: runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v4 + - uses: dessant/lock-threads@v5 with: github-token: ${{ secrets.GITHUB_TOKEN }} issue-comment: > diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index cb32a0f815..3973df4438 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -14,7 +14,7 @@ jobs: steps: # Please look up the latest version from # https://github.com/amannn/action-semantic-pull-request/releases - - uses: amannn/action-semantic-pull-request@v5.0.2 + - uses: amannn/action-semantic-pull-request@v5.4.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index cb82671340..c2632d1a44 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -8,7 +8,7 @@ on: env: TERRAFORM_DOCS_VERSION: v0.16.0 - TFLINT_VERSION: v0.44.1 + TFLINT_VERSION: v0.50.3 jobs: collectInputs: @@ -18,11 +18,11 @@ jobs: directories: ${{ steps.dirs.outputs.directories }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get root directories id: dirs - uses: clowdhaus/terraform-composite-actions/directories@v1.8.3 + uses: clowdhaus/terraform-composite-actions/directories@v1.9.0 preCommitMinVersions: name: Min TF pre-commit @@ -32,19 +32,27 @@ jobs: matrix: directory: ${{ fromJson(needs.collectInputs.outputs.directories) }} steps: + # https://github.com/orgs/community/discussions/25678#discussioncomment-5242449 + - name: Delete huge unnecessary tools folder + run: | + rm -rf /opt/hostedtoolcache/CodeQL + rm -rf /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk + rm -rf /opt/hostedtoolcache/Ruby + rm -rf /opt/hostedtoolcache/go + - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.2.4 + uses: clowdhaus/terraform-min-max@v1.3.0 with: directory: ${{ matrix.directory }} - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory != '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.9.0 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} tflint-version: ${{ env.TFLINT_VERSION }} @@ -53,7 +61,7 @@ jobs: - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory == '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.9.0 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} tflint-version: ${{ env.TFLINT_VERSION }} @@ -64,18 +72,26 @@ jobs: runs-on: ubuntu-latest needs: collectInputs steps: + # https://github.com/orgs/community/discussions/25678#discussioncomment-5242449 + - name: Delete huge unnecessary tools folder + run: | + rm -rf /opt/hostedtoolcache/CodeQL + rm -rf /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk + rm -rf /opt/hostedtoolcache/Ruby + rm -rf /opt/hostedtoolcache/go + - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.ref }} repository: ${{github.event.pull_request.head.repo.full_name}} - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.2.4 + uses: clowdhaus/terraform-min-max@v1.3.0 - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.9.0 with: terraform-version: ${{ steps.minMax.outputs.maxVersion }} tflint-version: ${{ env.TFLINT_VERSION }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 81f674740b..4a9422614e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,18 +20,18 @@ jobs: if: github.repository_owner == 'terraform-aws-modules' steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false fetch-depth: 0 - name: Release - uses: cycjimmy/semantic-release-action@v3 + uses: cycjimmy/semantic-release-action@v4 with: - semantic_version: 18.0.0 + semantic_version: 23.0.2 extra_plugins: | - @semantic-release/changelog@6.0.0 - @semantic-release/git@10.0.0 - conventional-changelog-conventionalcommits@4.6.3 + @semantic-release/changelog@6.0.3 + @semantic-release/git@10.0.1 + conventional-changelog-conventionalcommits@7.0.2 env: GITHUB_TOKEN: ${{ secrets.SEMANTIC_RELEASE_TOKEN }} diff --git a/.github/workflows/stale-actions.yaml b/.github/workflows/stale-actions.yaml index 50379957fd..6ccd0ed856 100644 --- a/.github/workflows/stale-actions.yaml +++ b/.github/workflows/stale-actions.yaml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v6 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Staling issues and PR's diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c542c8442..395404e805 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,6 @@ repos: rev: v1.88.0 hooks: - id: terraform_fmt - - id: terraform_validate - id: terraform_docs args: - '--args=--lockfile=false' @@ -22,6 +21,8 @@ repos: - '--args=--only=terraform_required_providers' - '--args=--only=terraform_standard_module_structure' - '--args=--only=terraform_workspace_remote' + - '--args=--only=terraform_unused_required_providers' + - id: terraform_validate - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: From 71b5031830015722531df4e8b171ecea1e284e81 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 7 Mar 2024 02:31:02 +0000 Subject: [PATCH 750/962] chore(release): version 20.5.1 [skip ci] ## [20.5.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.0...v20.5.1) (2024-03-07) ### Bug Fixes * Update CI workflow versions to remove deprecated runtime warnings ([#2956](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2956)) ([d14cc92](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d14cc925c450451b023407d05a2516d7682d1617)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aae04abad5..bba834a3e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.5.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.0...v20.5.1) (2024-03-07) + + +### Bug Fixes + +* Update CI workflow versions to remove deprecated runtime warnings ([#2956](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2956)) ([d14cc92](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d14cc925c450451b023407d05a2516d7682d1617)) + ## [20.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.4.0...v20.5.0) (2024-03-01) From 0ed32d7b291513f34775ca85b0aa33da085d09fa Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 7 Mar 2024 14:40:43 -0500 Subject: [PATCH 751/962] fix: Use the `launch_template_tags` on the launch template (#2957) --- examples/eks_managed_node_group/main.tf | 6 ++++++ modules/eks-managed-node-group/main.tf | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 17ff2bbc80..e464ca73ca 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -264,6 +264,12 @@ module "eks" { additional = aws_iam_policy.node_additional.arn } + launch_template_tags = { + # enable discovery of autoscaling groups by cluster-autoscaler + "k8s.io/cluster-autoscaler/enabled" : true, + "k8s.io/cluster-autoscaler/${local.name}" : "owned", + } + tags = { ExtraTag = "EKS managed node group complete example" } diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index fa74cd3b76..75ee289884 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -312,7 +312,10 @@ resource "aws_launch_template" "this" { user_data = module.user_data.user_data vpc_security_group_ids = length(local.network_interfaces) > 0 ? [] : local.security_group_ids - tags = var.tags + tags = merge( + var.tags, + var.launch_template_tags, + ) # Prevent premature access of policies by pods that # require permissions on create/destroy that depend on nodes From e547957e0dc39fc7110b7527fee23ccd02b14496 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 7 Mar 2024 19:41:24 +0000 Subject: [PATCH 752/962] chore(release): version 20.5.2 [skip ci] ## [20.5.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.1...v20.5.2) (2024-03-07) ### Bug Fixes * Use the `launch_template_tags` on the launch template ([#2957](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2957)) ([0ed32d7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0ed32d7b291513f34775ca85b0aa33da085d09fa)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bba834a3e5..0cff6f6af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.5.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.1...v20.5.2) (2024-03-07) + + +### Bug Fixes + +* Use the `launch_template_tags` on the launch template ([#2957](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2957)) ([0ed32d7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0ed32d7b291513f34775ca85b0aa33da085d09fa)) + ## [20.5.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.0...v20.5.1) (2024-03-07) From dfe41141c2385db783d97494792c8f2e227cfc7c Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 8 Mar 2024 15:25:14 -0500 Subject: [PATCH 753/962] fix: Update AWS provider version to support `AL2023_*` AMI types; ensure AL2023 user data receives cluster service CIDR (#2960) * fix: Update AWS provider version to support `AL2023_*` AMI types * fix: Ensure the cluster service CIDR is passed to the AL2023 user data where its required --- README.md | 4 +-- examples/eks_managed_node_group/README.md | 4 +-- examples/eks_managed_node_group/main.tf | 1 + examples/eks_managed_node_group/versions.tf | 2 +- examples/fargate_profile/README.md | 4 +-- examples/fargate_profile/versions.tf | 2 +- examples/karpenter/README.md | 6 ++-- examples/karpenter/versions.tf | 2 +- examples/outposts/README.md | 4 +-- examples/outposts/prerequisites/versions.tf | 2 +- examples/outposts/versions.tf | 2 +- examples/self_managed_node_group/README.md | 4 +-- examples/self_managed_node_group/versions.tf | 2 +- examples/user_data/main.tf | 31 ++++++++++++------- .../rendered/al2023/eks-mng-custom-ami.sh | 2 +- .../al2023/eks-mng-custom-template.sh | 1 + .../rendered/al2023/self-mng-bootstrap.sh | 1 + .../al2023/self-mng-custom-template.sh | 1 + .../user_data/templates/al2023_custom.tpl | 1 + modules/_user_data/README.md | 1 + modules/_user_data/main.tf | 3 ++ modules/_user_data/variables.tf | 8 +++++ modules/eks-managed-node-group/README.md | 5 +-- modules/eks-managed-node-group/main.tf | 3 +- modules/eks-managed-node-group/variables.tf | 8 +++++ modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 +-- modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 4 +-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 5 +-- modules/self-managed-node-group/main.tf | 7 +++-- modules/self-managed-node-group/variables.tf | 7 +++++ modules/self-managed-node-group/versions.tf | 2 +- node_groups.tf | 9 ++++-- templates/al2023_user_data.tpl | 4 +-- versions.tf | 2 +- 37 files changed, 99 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 1b763ec7de..a3fbd76bf6 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.38 | +| [aws](#requirement\_aws) | >= 5.40 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -197,7 +197,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.38 | +| [aws](#provider\_aws) | >= 5.40 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 1772d1324a..f57a972247 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -30,13 +30,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.38 | +| [aws](#requirement\_aws) | >= 5.40 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.38 | +| [aws](#provider\_aws) | >= 5.40 | ## Modules diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index e464ca73ca..f2f8b13875 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -89,6 +89,7 @@ module "eks" { # AL2023 node group utilizing new user data format which utilizes nodeadm # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh) al2023_nodeadm = { + ami_type = "AL2023_x86_64_STANDARD" platform = "al2023" cloudinit_pre_nodeadm = [ diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index fb651ab50d..876adbdfa4 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } } } diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 55e303ce15..6f33b44cd2 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -20,13 +20,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.38 | +| [aws](#requirement\_aws) | >= 5.40 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.38 | +| [aws](#provider\_aws) | >= 5.40 | ## Modules diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index fb651ab50d..876adbdfa4 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index f36f22c2e5..97d5f963bf 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -55,7 +55,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.38 | +| [aws](#requirement\_aws) | >= 5.40 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -63,8 +63,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.38 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.38 | +| [aws](#provider\_aws) | >= 5.40 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.40 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 711da76a33..9cdb8122b9 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } helm = { source = "hashicorp/helm" diff --git a/examples/outposts/README.md b/examples/outposts/README.md index 87bdafa73a..e1daea0b83 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -49,14 +49,14 @@ terraform destroy | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.38 | +| [aws](#requirement\_aws) | >= 5.40 | | [kubernetes](#requirement\_kubernetes) | >= 2.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.38 | +| [aws](#provider\_aws) | >= 5.40 | | [kubernetes](#provider\_kubernetes) | >= 2.20 | ## Modules diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf index fb651ab50d..876adbdfa4 100644 --- a/examples/outposts/prerequisites/versions.tf +++ b/examples/outposts/prerequisites/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } } } diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf index 73a045bb35..6ff0747487 100644 --- a/examples/outposts/versions.tf +++ b/examples/outposts/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 2566c3d9cb..7aea278b77 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -26,13 +26,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.38 | +| [aws](#requirement\_aws) | >= 5.40 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.38 | +| [aws](#provider\_aws) | >= 5.40 | ## Modules diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index fb651ab50d..876adbdfa4 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } } } diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index 6a90081ec0..35b15302ba 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -4,6 +4,7 @@ locals { cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" cluster_service_ipv4_cidr = "172.16.0.0/16" + cluster_service_cidr = "192.168.0.0/16" } ################################################################################ @@ -100,9 +101,12 @@ module "eks_mng_al2023_custom_ami" { platform = "al2023" - cluster_name = local.name - cluster_endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_cidr = local.cluster_service_cidr + + # Should do nothing cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr enable_bootstrap_user_data = true @@ -135,9 +139,10 @@ module "eks_mng_al2023_custom_template" { platform = "al2023" - cluster_name = local.name - cluster_endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_cidr = local.cluster_service_cidr enable_bootstrap_user_data = true user_data_template_path = "${path.module}/templates/al2023_custom.tpl" @@ -371,9 +376,10 @@ module "self_mng_al2023_bootstrap" { enable_bootstrap_user_data = true is_eks_managed_node_group = false - cluster_name = local.name - cluster_endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_cidr = local.cluster_service_cidr cloudinit_pre_nodeadm = [{ content = <<-EOT @@ -406,9 +412,10 @@ module "self_mng_al2023_custom_template" { enable_bootstrap_user_data = true is_eks_managed_node_group = false - cluster_name = local.name - cluster_endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_cidr = local.cluster_service_cidr user_data_template_path = "${path.module}/templates/al2023_custom.tpl" diff --git a/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh b/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh index fe8f07a24b..676edd1df6 100755 --- a/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh +++ b/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh @@ -29,7 +29,7 @@ spec: name: ex-user-data apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== - cidr: 172.16.0.0/16 + cidr: 192.168.0.0/16 --MIMEBOUNDARY Content-Transfer-Encoding: 7bit diff --git a/examples/user_data/rendered/al2023/eks-mng-custom-template.sh b/examples/user_data/rendered/al2023/eks-mng-custom-template.sh index 6267b85f72..3fe97563e2 100755 --- a/examples/user_data/rendered/al2023/eks-mng-custom-template.sh +++ b/examples/user_data/rendered/al2023/eks-mng-custom-template.sh @@ -29,6 +29,7 @@ spec: name: ex-user-data apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + cidr: 192.168.0.0/16 containerd: config: | [plugins."io.containerd.grpc.v1.cri".containerd] diff --git a/examples/user_data/rendered/al2023/self-mng-bootstrap.sh b/examples/user_data/rendered/al2023/self-mng-bootstrap.sh index 9ea008476e..676edd1df6 100755 --- a/examples/user_data/rendered/al2023/self-mng-bootstrap.sh +++ b/examples/user_data/rendered/al2023/self-mng-bootstrap.sh @@ -29,6 +29,7 @@ spec: name: ex-user-data apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + cidr: 192.168.0.0/16 --MIMEBOUNDARY Content-Transfer-Encoding: 7bit diff --git a/examples/user_data/rendered/al2023/self-mng-custom-template.sh b/examples/user_data/rendered/al2023/self-mng-custom-template.sh index 6267b85f72..3fe97563e2 100755 --- a/examples/user_data/rendered/al2023/self-mng-custom-template.sh +++ b/examples/user_data/rendered/al2023/self-mng-custom-template.sh @@ -29,6 +29,7 @@ spec: name: ex-user-data apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + cidr: 192.168.0.0/16 containerd: config: | [plugins."io.containerd.grpc.v1.cri".containerd] diff --git a/examples/user_data/templates/al2023_custom.tpl b/examples/user_data/templates/al2023_custom.tpl index a33aa44bf2..34c566c154 100644 --- a/examples/user_data/templates/al2023_custom.tpl +++ b/examples/user_data/templates/al2023_custom.tpl @@ -7,6 +7,7 @@ spec: name: ${cluster_name} apiServerEndpoint: ${cluster_endpoint} certificateAuthority: ${cluster_auth_base64} + cidr: ${cluster_service_cidr} containerd: config: | [plugins."io.containerd.grpc.v1.cri".containerd] diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 89edf259a5..541f2f880f 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -39,6 +39,7 @@ No modules. | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | +| [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [create](#input\_create) | Determines whether to create user-data or not | `bool` | `true` | no | | [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template | `bool` | `false` | no | diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index b70279bd31..ad3c4e2488 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -18,6 +18,9 @@ locals { cluster_endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 + # Required by AL2023 + cluster_service_cidr = var.cluster_service_cidr + # Optional cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr != null ? var.cluster_service_ipv4_cidr : "" bootstrap_extra_args = var.bootstrap_extra_args diff --git a/modules/_user_data/variables.tf b/modules/_user_data/variables.tf index 96c1b07059..d3d8082dbd 100644 --- a/modules/_user_data/variables.tf +++ b/modules/_user_data/variables.tf @@ -40,6 +40,14 @@ variable "cluster_auth_base64" { default = "" } +# Currently only used by AL2023 since it can be IPv4 or IPv6 +variable "cluster_service_cidr" { + description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" + type = string + default = "" +} + +# Not used by AL2023 variable "cluster_service_ipv4_cidr" { description = "The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" type = string diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 708f7ad1f6..76852373da 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.38 | +| [aws](#requirement\_aws) | >= 5.40 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.38 | +| [aws](#provider\_aws) | >= 5.40 | ## Modules @@ -112,6 +112,7 @@ module "eks_managed_node_group" { | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | | [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | +| [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes version. Defaults to EKS Cluster Kubernetes version | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 75ee289884..3e9ce1d07e 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -16,6 +16,7 @@ module "user_data" { cluster_auth_base64 = var.cluster_auth_base64 cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr + cluster_service_cidr = var.cluster_service_cidr enable_bootstrap_user_data = var.enable_bootstrap_user_data pre_bootstrap_user_data = var.pre_bootstrap_user_data @@ -29,7 +30,7 @@ module "user_data" { ################################################################################ data "aws_ec2_instance_type" "this" { - count = var.enable_efa_support ? 1 : 0 + count = var.create && var.enable_efa_support ? 1 : 0 instance_type = local.efa_instance_type } diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 192980888a..eb4cc86b56 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -44,6 +44,14 @@ variable "cluster_auth_base64" { default = "" } +# Currently only used by AL2023 since it can be IPv4 or IPv6 +variable "cluster_service_cidr" { + description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" + type = string + default = "" +} + +# Not used by AL2023 variable "cluster_service_ipv4_cidr" { description = "The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" type = string diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index fb651ab50d..876adbdfa4 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 1fb59c4ba0..fb4e44a4d6 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.38 | +| [aws](#requirement\_aws) | >= 5.40 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.38 | +| [aws](#provider\_aws) | >= 5.40 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index fb651ab50d..876adbdfa4 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index d6f27bfd1f..5faac77cf7 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -85,13 +85,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.38 | +| [aws](#requirement\_aws) | >= 5.40 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.38 | +| [aws](#provider\_aws) | >= 5.40 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index fb651ab50d..876adbdfa4 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 6fe6e43180..be861a0cd9 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.38 | +| [aws](#requirement\_aws) | >= 5.40 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.38 | +| [aws](#provider\_aws) | >= 5.40 | ## Modules @@ -94,6 +94,7 @@ module "self_managed_node_group" { | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `""` | no | | [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | +| [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | | [context](#input\_context) | Reserved | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index bb1eea0b54..8b0f75a70d 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -24,9 +24,10 @@ module "user_data" { platform = var.platform is_eks_managed_node_group = false - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint - cluster_auth_base64 = var.cluster_auth_base64 + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + cluster_service_cidr = var.cluster_service_cidr enable_bootstrap_user_data = true pre_bootstrap_user_data = var.pre_bootstrap_user_data diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 54bded55fe..0449678f26 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -38,6 +38,13 @@ variable "cluster_auth_base64" { default = "" } +# Currently only used by AL2023 since it can be IPv4 or IPv6 +variable "cluster_service_cidr" { + description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" + type = string + default = "" +} + variable "pre_bootstrap_user_data" { description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" type = string diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index fb651ab50d..876adbdfa4 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } } } diff --git a/node_groups.tf b/node_groups.tf index a225bb8931..34a0735ff1 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -30,9 +30,10 @@ resource "time_sleep" "this" { create_duration = var.dataplane_wait_duration triggers = { - cluster_name = aws_eks_cluster.this[0].name - cluster_endpoint = aws_eks_cluster.this[0].endpoint - cluster_version = aws_eks_cluster.this[0].version + cluster_name = aws_eks_cluster.this[0].name + cluster_endpoint = aws_eks_cluster.this[0].endpoint + cluster_version = aws_eks_cluster.this[0].version + cluster_service_cidr = var.cluster_ip_family == "ipv6" ? aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv6_cidr : aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv4_cidr cluster_certificate_authority_data = aws_eks_cluster.this[0].certificate_authority[0].data } @@ -329,6 +330,7 @@ module "eks_managed_node_group" { cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr + cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") enable_bootstrap_user_data = try(each.value.enable_bootstrap_user_data, var.eks_managed_node_group_defaults.enable_bootstrap_user_data, false) pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.eks_managed_node_group_defaults.pre_bootstrap_user_data, "") post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.eks_managed_node_group_defaults.post_bootstrap_user_data, "") @@ -460,6 +462,7 @@ module "self_managed_node_group" { platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, "linux") cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") + cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.self_managed_node_group_defaults.pre_bootstrap_user_data, "") post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "") bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "") diff --git a/templates/al2023_user_data.tpl b/templates/al2023_user_data.tpl index 820223f719..cc360e6d65 100644 --- a/templates/al2023_user_data.tpl +++ b/templates/al2023_user_data.tpl @@ -7,7 +7,5 @@ spec: name: ${cluster_name} apiServerEndpoint: ${cluster_endpoint} certificateAuthority: ${cluster_auth_base64} -%{ if length(cluster_service_ipv4_cidr) > 0 ~} - cidr: ${cluster_service_ipv4_cidr} -%{ endif ~} + cidr: ${cluster_service_cidr} %{ endif ~} diff --git a/versions.tf b/versions.tf index 4d006cbfb6..f759deeef2 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.38" + version = ">= 5.40" } tls = { source = "hashicorp/tls" From f1bbfc4740d2fc16fb68310fbf00a3bd47163aa9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 8 Mar 2024 20:25:43 +0000 Subject: [PATCH 754/962] chore(release): version 20.5.3 [skip ci] ## [20.5.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.2...v20.5.3) (2024-03-08) ### Bug Fixes * Update AWS provider version to support `AL2023_*` AMI types; ensure AL2023 user data receives cluster service CIDR ([#2960](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2960)) ([dfe4114](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/dfe41141c2385db783d97494792c8f2e227cfc7c)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cff6f6af8..32ccefb89e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.5.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.2...v20.5.3) (2024-03-08) + + +### Bug Fixes + +* Update AWS provider version to support `AL2023_*` AMI types; ensure AL2023 user data receives cluster service CIDR ([#2960](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2960)) ([dfe4114](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/dfe41141c2385db783d97494792c8f2e227cfc7c)) + ## [20.5.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.1...v20.5.2) (2024-03-07) From 393da7ec0ed158cf783356ab10959d91430c1d80 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 8 Mar 2024 22:48:38 -0500 Subject: [PATCH 755/962] feat: Add support for tracking latest AMI release version on managed nodegroups (#2951) --- examples/eks_managed_node_group/main.tf | 4 ++ modules/eks-managed-node-group/README.md | 2 + modules/eks-managed-node-group/main.tf | 41 ++++++++++++++++++++- modules/eks-managed-node-group/variables.tf | 6 +++ node_groups.tf | 7 ++-- 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index f2f8b13875..e31d2b2fb6 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -92,6 +92,8 @@ module "eks" { ami_type = "AL2023_x86_64_STANDARD" platform = "al2023" + use_latest_ami_release_version = true + cloudinit_pre_nodeadm = [ { content_type = "application/node.eks.aws" @@ -125,6 +127,8 @@ module "eks" { ami_type = "BOTTLEROCKET_x86_64" platform = "bottlerocket" + use_latest_ami_release_version = true + # This will get added to what AWS provides bootstrap_extra_args = <<-EOT # extra args added diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 76852373da..82c3515cee 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -94,6 +94,7 @@ module "eks_managed_node_group" { | [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_ssm_parameter.ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | | [aws_subnets.efa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs @@ -176,6 +177,7 @@ module "eks_managed_node_group" { | [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates | `map(string)` |
{
"max_unavailable_percentage": 33
}
| no | | [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update the launch templates default version on each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | | [use\_custom\_launch\_template](#input\_use\_custom\_launch\_template) | Determines whether to use a custom launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | +| [use\_latest\_ami\_release\_version](#input\_use\_latest\_ami\_release\_version) | Determines whether to use the latest AMI release version for the given `ami_type` (except for `CUSTOM`). Note: `ami_type` and `cluster_version` must be supplied in order to enable this feature | `bool` | `false` | no | | [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix | `bool` | `true` | no | | [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | | [vpc\_security\_group\_ids](#input\_vpc\_security\_group\_ids) | A list of security group IDs to associate | `list(string)` | `[]` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 3e9ce1d07e..b10b71e6b5 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -329,6 +329,45 @@ resource "aws_launch_template" "this" { } } +################################################################################ +# AMI SSM Parameter +################################################################################ + +locals { + # Just to ensure templating doesn't fail when values are not provided + ssm_cluster_version = var.cluster_version != null ? var.cluster_version : "" + ssm_ami_type = var.ami_type != null ? var.ami_type : "" + + # Map the AMI type to the respective SSM param path + ssm_ami_type_to_ssm_param = { + AL2_x86_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2/recommended/release_version" + AL2_x86_64_GPU = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-gpu/recommended/release_version" + AL2_ARM_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-arm64/recommended/release_version" + CUSTOM = "NONE" + BOTTLEROCKET_ARM_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/arm64/latest/image_version" + BOTTLEROCKET_x86_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/x86_64/latest/image_version" + BOTTLEROCKET_ARM_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/arm64/latest/image_version" + BOTTLEROCKET_x86_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/x86_64/latest/image_version" + WINDOWS_CORE_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-${local.ssm_cluster_version}" + WINDOWS_FULL_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-${local.ssm_cluster_version}" + WINDOWS_CORE_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-EKS_Optimized-${local.ssm_cluster_version}" + WINDOWS_FULL_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-${local.ssm_cluster_version}" + AL2023_x86_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/standard/recommended/release_version" + AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/standard/recommended/release_version" + } + + # The Windows SSM params currently do not have a release version, so we have to get the full output JSON blob and parse out the release version + windows_latest_ami_release_version = var.create && var.use_latest_ami_release_version && startswith(local.ssm_ami_type, "WINDOWS") ? nonsensitive(jsondecode(data.aws_ssm_parameter.ami[0].value)["release_version"]) : null + # Based on the steps above, try to get an AMI release version - if not, `null` is returned + latest_ami_release_version = startswith(local.ssm_ami_type, "WINDOWS") ? local.windows_latest_ami_release_version : try(nonsensitive(data.aws_ssm_parameter.ami[0].value), null) +} + +data "aws_ssm_parameter" "ami" { + count = var.create && var.use_latest_ami_release_version ? 1 : 0 + + name = local.ssm_ami_type_to_ssm_param[var.ami_type] +} + ################################################################################ # Node Group ################################################################################ @@ -359,7 +398,7 @@ resource "aws_eks_node_group" "this" { # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami ami_type = var.ami_id != "" ? null : var.ami_type - release_version = var.ami_id != "" ? null : var.ami_release_version + release_version = var.ami_id != "" ? null : var.use_latest_ami_release_version ? local.latest_ami_release_version : var.ami_release_version version = var.ami_id != "" ? null : var.cluster_version capacity_type = var.capacity_type diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index eb4cc86b56..b5494e9dda 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -346,6 +346,12 @@ variable "ami_release_version" { default = null } +variable "use_latest_ami_release_version" { + description = "Determines whether to use the latest AMI release version for the given `ami_type` (except for `CUSTOM`). Note: `ami_type` and `cluster_version` must be supplied in order to enable this feature" + type = bool + default = false +} + variable "capacity_type" { description = "Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT`" type = string diff --git a/node_groups.tf b/node_groups.tf index 34a0735ff1..457e903d64 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -310,9 +310,10 @@ module "eks_managed_node_group" { max_size = try(each.value.max_size, var.eks_managed_node_group_defaults.max_size, 3) desired_size = try(each.value.desired_size, var.eks_managed_node_group_defaults.desired_size, 1) - ami_id = try(each.value.ami_id, var.eks_managed_node_group_defaults.ami_id, "") - ami_type = try(each.value.ami_type, var.eks_managed_node_group_defaults.ami_type, null) - ami_release_version = try(each.value.ami_release_version, var.eks_managed_node_group_defaults.ami_release_version, null) + ami_id = try(each.value.ami_id, var.eks_managed_node_group_defaults.ami_id, "") + ami_type = try(each.value.ami_type, var.eks_managed_node_group_defaults.ami_type, null) + ami_release_version = try(each.value.ami_release_version, var.eks_managed_node_group_defaults.ami_release_version, null) + use_latest_ami_release_version = try(each.value.use_latest_ami_release_version, var.eks_managed_node_group_defaults.use_latest_ami_release_version, false) capacity_type = try(each.value.capacity_type, var.eks_managed_node_group_defaults.capacity_type, null) disk_size = try(each.value.disk_size, var.eks_managed_node_group_defaults.disk_size, null) From 0be0a99b1a323f6df0888703456c2505ce4ec494 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 9 Mar 2024 03:49:05 +0000 Subject: [PATCH 756/962] chore(release): version 20.6.0 [skip ci] ## [20.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.3...v20.6.0) (2024-03-09) ### Features * Add support for tracking latest AMI release version on managed nodegroups ([#2951](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2951)) ([393da7e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/393da7ec0ed158cf783356ab10959d91430c1d80)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32ccefb89e..0cd00e4e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.3...v20.6.0) (2024-03-09) + + +### Features + +* Add support for tracking latest AMI release version on managed nodegroups ([#2951](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2951)) ([393da7e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/393da7ec0ed158cf783356ab10959d91430c1d80)) + ## [20.5.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.2...v20.5.3) (2024-03-08) From 30316312f33fe7fd09faf86fdb1b01ab2a377b2a Mon Sep 17 00:00:00 2001 From: James Masson Date: Sat, 9 Mar 2024 23:34:50 +0000 Subject: [PATCH 757/962] feat: Add supprot for creating placement group for managed node group (#2959) Co-authored-by: Bryant Biggs --- modules/eks-managed-node-group/README.md | 2 ++ modules/eks-managed-node-group/main.tf | 6 +++--- modules/eks-managed-node-group/variables.tf | 12 ++++++++++++ node_groups.tf | 2 ++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 82c3515cee..a0ca56ee7b 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -120,6 +120,7 @@ module "eks_managed_node_group" { | [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | +| [create\_placement\_group](#input\_create\_placement\_group) | Determines whether a placement group is created & used by the nodegroup | `bool` | `false` | no | | [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | | [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | | [desired\_size](#input\_desired\_size) | Desired number of instances/nodes | `number` | `1` | no | @@ -162,6 +163,7 @@ module "eks_managed_node_group" { | [name](#input\_name) | Name of the EKS managed node group | `string` | `""` | no | | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | | [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | +| [placement\_group\_strategy](#input\_placement\_group\_strategy) | The placement group strategy | `string` | `"cluster"` | no | | [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket` or `linux` based; `windows` is not supported | `string` | `"linux"` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | | [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index b10b71e6b5..dd924be535 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -60,7 +60,7 @@ locals { launch_template_name = coalesce(var.launch_template_name, "${var.name}-eks-node-group") security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) - placement = var.create && var.enable_efa_support ? { group_name = aws_placement_group.this[0].name } : var.placement + placement = var.create && (var.enable_efa_support || var.create_placement_group) ? { group_name = aws_placement_group.this[0].name } : var.placement } resource "aws_launch_template" "this" { @@ -526,10 +526,10 @@ resource "aws_iam_role_policy_attachment" "additional" { ################################################################################ resource "aws_placement_group" "this" { - count = var.create && var.enable_efa_support ? 1 : 0 + count = var.create && (var.enable_efa_support || var.create_placement_group) ? 1 : 0 name = "${var.cluster_name}-${var.name}" - strategy = "cluster" + strategy = var.placement_group_strategy tags = var.tags } diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index b5494e9dda..101e20b584 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -276,6 +276,18 @@ variable "placement" { default = {} } +variable "create_placement_group" { + description = "Determines whether a placement group is created & used by the nodegroup" + type = bool + default = false +} + +variable "placement_group_strategy" { + description = "The placement group strategy" + type = string + default = "cluster" +} + variable "private_dns_name_options" { description = "The options for the instance hostname. The default values are inherited from the subnet" type = map(string) diff --git a/node_groups.tf b/node_groups.tf index 457e903d64..26d5ed7ec8 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -369,6 +369,8 @@ module "eks_managed_node_group" { metadata_options = try(each.value.metadata_options, var.eks_managed_node_group_defaults.metadata_options, local.metadata_options) enable_monitoring = try(each.value.enable_monitoring, var.eks_managed_node_group_defaults.enable_monitoring, true) enable_efa_support = try(each.value.enable_efa_support, var.eks_managed_node_group_defaults.enable_efa_support, false) + create_placement_group = try(each.value.create_placement_group, var.eks_managed_node_group_defaults.create_placement_group, false) + placement_group_strategy = try(each.value.placement_group_strategy, var.eks_managed_node_group_defaults.placement_group_strategy, "cluster") network_interfaces = try(each.value.network_interfaces, var.eks_managed_node_group_defaults.network_interfaces, []) placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, {}) maintenance_options = try(each.value.maintenance_options, var.eks_managed_node_group_defaults.maintenance_options, {}) From bed26ac4216f64cf71b9e50fbe7c40469fead396 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 9 Mar 2024 23:35:19 +0000 Subject: [PATCH 758/962] chore(release): version 20.7.0 [skip ci] ## [20.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.6.0...v20.7.0) (2024-03-09) ### Features * Add supprot for creating placement group for managed node group ([#2959](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2959)) ([3031631](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/30316312f33fe7fd09faf86fdb1b01ab2a377b2a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cd00e4e1b..467cf71772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.6.0...v20.7.0) (2024-03-09) + + +### Features + +* Add supprot for creating placement group for managed node group ([#2959](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2959)) ([3031631](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/30316312f33fe7fd09faf86fdb1b01ab2a377b2a)) + ## [20.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.5.3...v20.6.0) (2024-03-09) From 57f5130132ca11fd3e478a61a8fc082a929540c2 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 10 Mar 2024 12:49:41 -0400 Subject: [PATCH 759/962] feat: Replace the use of `toset()` with static keys for node IAM role policy attachment (#2962) --- modules/eks-managed-node-group/main.tf | 11 +++++----- modules/eks-managed-node-group/migrations.tf | 20 ++++++++++++++++++ modules/karpenter/main.tf | 11 +++++----- modules/karpenter/migrations.tf | 21 +++++++++++++++++++ modules/self-managed-node-group/main.tf | 12 ++++++----- modules/self-managed-node-group/migrations.tf | 20 ++++++++++++++++++ 6 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 modules/eks-managed-node-group/migrations.tf create mode 100644 modules/self-managed-node-group/migrations.tf diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index dd924be535..10b7fbe678 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -504,11 +504,12 @@ resource "aws_iam_role" "this" { # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group resource "aws_iam_role_policy_attachment" "this" { - for_each = { for k, v in toset(compact([ - "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", - "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", - var.iam_role_attach_cni_policy ? local.cni_policy : "", - ])) : k => v if var.create && var.create_iam_role } + for_each = { for k, v in { + AmazonEKSWorkerNodePolicy = "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" + AmazonEC2ContainerRegistryReadOnly = "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" + AmazonEKS_CNI_IPv6_Policy = var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" ? local.cni_policy : "" + AmazonEKS_CNI_Policy = var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" ? local.cni_policy : "" + } : k => v if var.create && var.create_iam_role && v != "" } policy_arn = each.value role = aws_iam_role.this[0].name diff --git a/modules/eks-managed-node-group/migrations.tf b/modules/eks-managed-node-group/migrations.tf new file mode 100644 index 0000000000..5d51a7208a --- /dev/null +++ b/modules/eks-managed-node-group/migrations.tf @@ -0,0 +1,20 @@ +################################################################################ +# Migrations: v20.7 -> v20.8 +################################################################################ + +# Node IAM role policy attachment +# Commercial partition only - `moved` does now allow multiple moves to same target +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"] + to = aws_iam_role_policy_attachment.this["AmazonEKSWorkerNodePolicy"] +} + +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"] + to = aws_iam_role_policy_attachment.this["AmazonEC2ContainerRegistryReadOnly"] +} + +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"] + to = aws_iam_role_policy_attachment.this["AmazonEKS_CNI_Policy"] +} diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 4d01d77a01..719ceacc6e 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -565,11 +565,12 @@ resource "aws_iam_role" "node" { # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group resource "aws_iam_role_policy_attachment" "node" { - for_each = { for k, v in toset(compact([ - "${local.node_iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", - "${local.node_iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", - var.node_iam_role_attach_cni_policy ? local.cni_policy : "", - ])) : k => v if local.create_node_iam_role } + for_each = { for k, v in { + AmazonEKSWorkerNodePolicy = "${local.node_iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" + AmazonEC2ContainerRegistryReadOnly = "${local.node_iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" + AmazonEKS_CNI_IPv6_Policy = var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" ? local.cni_policy : "" + AmazonEKS_CNI_Policy = var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" ? local.cni_policy : "" + } : k => v if var.create && var.create_iam_role && v != "" } policy_arn = each.value role = aws_iam_role.node[0].name diff --git a/modules/karpenter/migrations.tf b/modules/karpenter/migrations.tf index 3e7ca59a9b..b40040f330 100644 --- a/modules/karpenter/migrations.tf +++ b/modules/karpenter/migrations.tf @@ -54,3 +54,24 @@ moved { from = aws_cloudwatch_event_rule.this["spot_interupt"] to = aws_cloudwatch_event_rule.this["spot_interrupt"] } + +################################################################################ +# Migrations: v20.7 -> v20.8 +################################################################################ + +# Node IAM role policy attachment +# Commercial partition only - `moved` does now allow multiple moves to same target +moved { + from = aws_iam_role_policy_attachment.node["arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"] + to = aws_iam_role_policy_attachment.node["AmazonEKSWorkerNodePolicy"] +} + +moved { + from = aws_iam_role_policy_attachment.node["arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"] + to = aws_iam_role_policy_attachment.node["AmazonEC2ContainerRegistryReadOnly"] +} + +moved { + from = aws_iam_role_policy_attachment.node["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"] + to = aws_iam_role_policy_attachment.node["AmazonEKS_CNI_Policy"] +} diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 8b0f75a70d..9bd1f6a9b5 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -768,12 +768,14 @@ resource "aws_iam_role" "this" { tags = merge(var.tags, var.iam_role_tags) } +# Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group resource "aws_iam_role_policy_attachment" "this" { - for_each = { for k, v in toset(compact([ - "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", - "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", - var.iam_role_attach_cni_policy ? local.cni_policy : "", - ])) : k => v if var.create && var.create_iam_instance_profile } + for_each = { for k, v in { + AmazonEKSWorkerNodePolicy = "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" + AmazonEC2ContainerRegistryReadOnly = "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" + AmazonEKS_CNI_IPv6_Policy = var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" ? local.cni_policy : "" + AmazonEKS_CNI_Policy = var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" ? local.cni_policy : "" + } : k => v if var.create && var.create_iam_instance_profile && v != "" } policy_arn = each.value role = aws_iam_role.this[0].name diff --git a/modules/self-managed-node-group/migrations.tf b/modules/self-managed-node-group/migrations.tf new file mode 100644 index 0000000000..5d51a7208a --- /dev/null +++ b/modules/self-managed-node-group/migrations.tf @@ -0,0 +1,20 @@ +################################################################################ +# Migrations: v20.7 -> v20.8 +################################################################################ + +# Node IAM role policy attachment +# Commercial partition only - `moved` does now allow multiple moves to same target +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"] + to = aws_iam_role_policy_attachment.this["AmazonEKSWorkerNodePolicy"] +} + +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"] + to = aws_iam_role_policy_attachment.this["AmazonEC2ContainerRegistryReadOnly"] +} + +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"] + to = aws_iam_role_policy_attachment.this["AmazonEKS_CNI_Policy"] +} From 6d3d6d9383076df7009e01e2bb79febcdd24ebf0 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 10 Mar 2024 16:50:16 +0000 Subject: [PATCH 760/962] chore(release): version 20.8.0 [skip ci] ## [20.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.7.0...v20.8.0) (2024-03-10) ### Features * Replace the use of `toset()` with static keys for node IAM role policy attachment ([#2962](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2962)) ([57f5130](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/57f5130132ca11fd3e478a61a8fc082a929540c2)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 467cf71772..6292ec3854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.7.0...v20.8.0) (2024-03-10) + + +### Features + +* Replace the use of `toset()` with static keys for node IAM role policy attachment ([#2962](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2962)) ([57f5130](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/57f5130132ca11fd3e478a61a8fc082a929540c2)) + ## [20.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.6.0...v20.7.0) (2024-03-09) From 3ad19d7435f34600e4872fd131e155583e498cd9 Mon Sep 17 00:00:00 2001 From: Piotr Roszatycki Date: Mon, 11 Mar 2024 00:16:24 +0100 Subject: [PATCH 761/962] fix: Do not attach policy if Karpenter node role is not created by module (#2964) --- modules/karpenter/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 719ceacc6e..6ce53f968b 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -570,7 +570,7 @@ resource "aws_iam_role_policy_attachment" "node" { AmazonEC2ContainerRegistryReadOnly = "${local.node_iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" AmazonEKS_CNI_IPv6_Policy = var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" ? local.cni_policy : "" AmazonEKS_CNI_Policy = var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" ? local.cni_policy : "" - } : k => v if var.create && var.create_iam_role && v != "" } + } : k => v if local.create_node_iam_role && v != "" } policy_arn = each.value role = aws_iam_role.node[0].name From a95487e5f2798ca54adc7b39d7ae7d1bcafed673 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 10 Mar 2024 23:16:59 +0000 Subject: [PATCH 762/962] chore(release): version 20.8.1 [skip ci] ## [20.8.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.0...v20.8.1) (2024-03-10) ### Bug Fixes * Do not attach policy if Karpenter node role is not created by module ([#2964](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2964)) ([3ad19d7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3ad19d7435f34600e4872fd131e155583e498cd9)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6292ec3854..530c26ed26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.8.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.0...v20.8.1) (2024-03-10) + + +### Bug Fixes + +* Do not attach policy if Karpenter node role is not created by module ([#2964](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2964)) ([3ad19d7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3ad19d7435f34600e4872fd131e155583e498cd9)) + ## [20.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.7.0...v20.8.0) (2024-03-10) From 29dcca335d80e248c57b8efa2c36aaef2e1b1bd2 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 11 Mar 2024 11:02:59 -0400 Subject: [PATCH 763/962] fix: Ensure a default `ip_family` value is provided to guarantee a CNI policy is attached to nodes (#2967) --- README.md | 2 +- examples/eks_managed_node_group/main.tf | 2 +- examples/karpenter/main.tf | 5 +++++ modules/eks-managed-node-group/README.md | 2 +- modules/eks-managed-node-group/variables.tf | 2 +- modules/fargate-profile/README.md | 2 +- modules/fargate-profile/variables.tf | 2 +- modules/karpenter/README.md | 2 +- modules/karpenter/variables.tf | 2 +- modules/self-managed-node-group/README.md | 2 +- modules/self-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/variables.tf | 2 +- variables.tf | 2 +- 13 files changed, 17 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a3fbd76bf6..a640a1ad51 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `false` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | -| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `null` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `"ipv4"` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | | [cluster\_security\_group\_additional\_rules](#input\_cluster\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created. Set `source_node_security_group = true` inside rules to set the `node_security_group` as source | `any` | `{}` | no | | [cluster\_security\_group\_description](#input\_cluster\_security\_group\_description) | Description of the cluster security group created | `string` | `"EKS cluster security group"` | no | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index e31d2b2fb6..e62b7a29b4 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -364,7 +364,7 @@ module "eks_managed_node_group" { subnet_ids = module.vpc.private_subnets cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id vpc_security_group_ids = [ - module.eks.cluster_security_group_id, + module.eks.node_security_group_id, ] ami_type = "BOTTLEROCKET_x86_64" diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index bd1540b3a0..e48f04af06 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -181,6 +181,11 @@ resource "helm_release" "karpenter" { serviceAccount: annotations: eks.amazonaws.com/role-arn: ${module.karpenter.iam_role_arn} + tolerations: + - key: 'eks.amazonaws.com/compute-type' + operator: Equal + value: fargate + effect: "NoSchedule" EOT ] } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index a0ca56ee7b..856b014ce7 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -110,7 +110,7 @@ module "eks_managed_node_group" { | [capacity\_type](#input\_capacity\_type) | Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT` | `string` | `"ON_DEMAND"` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | -| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | | [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | | [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 101e20b584..e0f8812df7 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -445,7 +445,7 @@ variable "create_iam_role" { variable "cluster_ip_family" { description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" type = string - default = null + default = "ipv4" } variable "iam_role_arn" { diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index fb4e44a4d6..5c2c882450 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -57,7 +57,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `null` | no | | [create](#input\_create) | Determines whether to create Fargate profile or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | diff --git a/modules/fargate-profile/variables.tf b/modules/fargate-profile/variables.tf index e22279dc6b..75816b0af8 100644 --- a/modules/fargate-profile/variables.tf +++ b/modules/fargate-profile/variables.tf @@ -23,7 +23,7 @@ variable "create_iam_role" { variable "cluster_ip_family" { description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" type = string - default = null + default = "ipv4" } variable "iam_role_arn" { diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 5faac77cf7..9d78640c21 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -128,7 +128,7 @@ No modules. |------|-------------|------|---------|:--------:| | [access\_entry\_type](#input\_access\_entry\_type) | Type of the access entry. `EC2_LINUX`, `FARGATE_LINUX`, or `EC2_WINDOWS`; defaults to `EC2_LINUX` | `string` | `"EC2_LINUX"` | no | | [ami\_id\_ssm\_parameter\_arns](#input\_ami\_id\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that Karpenter controller is allowed read access (for retrieving AMI IDs) | `list(string)` | `[]` | no | -| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. Note: If `ipv6` is specified, the `AmazonEKS_CNI_IPv6_Policy` must exist in the account. This policy is created by the EKS module with `create_cni_ipv6_iam_policy = true` | `string` | `null` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. Note: If `ipv6` is specified, the `AmazonEKS_CNI_IPv6_Policy` must exist in the account. This policy is created by the EKS module with `create_cni_ipv6_iam_policy = true` | `string` | `"ipv4"` | no | | [cluster\_name](#input\_cluster\_name) | The name of the EKS cluster | `string` | `""` | no | | [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the node IAM role | `bool` | `true` | no | diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 8d6bfe74ad..fc79b1a413 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -185,7 +185,7 @@ variable "create_node_iam_role" { variable "cluster_ip_family" { description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. Note: If `ipv6` is specified, the `AmazonEKS_CNI_IPv6_Policy` must exist in the account. This policy is created by the EKS module with `create_cni_ipv6_iam_policy = true`" type = string - default = null + default = "ipv4" } variable "node_iam_role_arn" { diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index be861a0cd9..eb39752b8c 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -91,7 +91,7 @@ module "self_managed_node_group" { | [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | -| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `""` | no | | [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | | [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 9bd1f6a9b5..c7e5ea6567 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -41,7 +41,7 @@ module "user_data" { ################################################################################ data "aws_ec2_instance_type" "this" { - count = var.enable_efa_support && local.instance_type_provided ? 1 : 0 + count = var.create && var.enable_efa_support && local.instance_type_provided ? 1 : 0 instance_type = var.instance_type } diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 0449678f26..7c93702e9f 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -549,7 +549,7 @@ variable "create_iam_instance_profile" { variable "cluster_ip_family" { description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" type = string - default = null + default = "ipv4" } variable "iam_instance_profile_arn" { diff --git a/variables.tf b/variables.tf index fff8895279..639110a9fc 100644 --- a/variables.tf +++ b/variables.tf @@ -83,7 +83,7 @@ variable "cluster_endpoint_public_access_cidrs" { variable "cluster_ip_family" { description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created" type = string - default = null + default = "ipv4" } variable "cluster_service_ipv4_cidr" { From 907f70cffdd03e14d1da97d916451cfb0688a760 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 11 Mar 2024 15:03:32 +0000 Subject: [PATCH 764/962] chore(release): version 20.8.2 [skip ci] ## [20.8.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.1...v20.8.2) (2024-03-11) ### Bug Fixes * Ensure a default `ip_family` value is provided to guarantee a CNI policy is attached to nodes ([#2967](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2967)) ([29dcca3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/29dcca335d80e248c57b8efa2c36aaef2e1b1bd2)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 530c26ed26..238dc4d58e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.8.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.1...v20.8.2) (2024-03-11) + + +### Bug Fixes + +* Ensure a default `ip_family` value is provided to guarantee a CNI policy is attached to nodes ([#2967](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2967)) ([29dcca3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/29dcca335d80e248c57b8efa2c36aaef2e1b1bd2)) + ## [20.8.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.0...v20.8.1) (2024-03-10) From aeb9f0c990b259320a6c3e5ff93be3f064bb9238 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 12 Mar 2024 10:36:19 -0400 Subject: [PATCH 765/962] fix: Ensure the correct service CIDR and IP family is used in the rendered user data (#2963) * fix: Ensuring the correct service CIDR and IP family is used in the rendered user data * chore: Updates from testing and validating * chore: Fix example destroy instructions * fix: Only require `cluster_service_cidr` when `create = true` * chore: Clean up commented out code and add note on check length --- .pre-commit-config.yaml | 2 +- README.md | 4 +- examples/eks_managed_node_group/README.md | 2 +- examples/eks_managed_node_group/main.tf | 7 +- examples/eks_managed_node_group/versions.tf | 2 +- examples/fargate_profile/README.md | 2 +- examples/fargate_profile/versions.tf | 2 +- examples/karpenter/README.md | 6 +- examples/karpenter/main.tf | 2 +- examples/karpenter/versions.tf | 2 +- examples/outposts/README.md | 2 +- examples/outposts/prerequisites/versions.tf | 2 +- examples/outposts/versions.tf | 2 +- examples/self_managed_node_group/README.md | 2 +- examples/self_managed_node_group/main.tf | 3 + examples/self_managed_node_group/versions.tf | 2 +- examples/user_data/README.md | 7 +- examples/user_data/main.tf | 153 ++++++++++++++---- examples/user_data/outputs.tf | 10 ++ .../rendered/al2/eks-mng-custom-ami-ipv6.sh | 8 + .../rendered/al2/eks-mng-custom-ami.sh | 4 +- .../rendered/al2/eks-mng-custom-template.sh | 3 +- .../rendered/al2/self-mng-bootstrap-ipv6.sh | 9 ++ .../rendered/al2/self-mng-bootstrap.sh | 3 +- .../rendered/al2/self-mng-custom-template.sh | 3 +- .../bottlerocket/eks-mng-custom-ami.toml | 1 + .../bottlerocket/self-mng-bootstrap.toml | 1 + examples/user_data/templates/linux_custom.tpl | 3 +- examples/user_data/versions.tf | 2 +- modules/_user_data/README.md | 8 +- modules/_user_data/main.tf | 29 +++- modules/_user_data/variables.tf | 11 +- modules/_user_data/versions.tf | 6 +- modules/aws-auth/README.md | 2 +- modules/aws-auth/versions.tf | 2 +- modules/eks-managed-node-group/README.md | 4 +- modules/eks-managed-node-group/main.tf | 41 +++-- modules/eks-managed-node-group/variables.tf | 5 +- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 2 +- modules/fargate-profile/main.tf | 27 +++- modules/fargate-profile/migrations.tf | 15 ++ modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 2 +- modules/karpenter/main.tf | 22 ++- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 2 +- modules/self-managed-node-group/main.tf | 49 +++--- modules/self-managed-node-group/variables.tf | 13 +- modules/self-managed-node-group/versions.tf | 2 +- node_groups.tf | 14 +- outputs.tf | 10 ++ templates/bottlerocket_user_data.tpl | 1 + templates/linux_user_data.tpl | 6 +- versions.tf | 2 +- 55 files changed, 384 insertions(+), 148 deletions(-) create mode 100755 examples/user_data/rendered/al2/eks-mng-custom-ami-ipv6.sh create mode 100755 examples/user_data/rendered/al2/self-mng-bootstrap-ipv6.sh create mode 100644 modules/fargate-profile/migrations.tf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 395404e805..9eda307037 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.88.0 + rev: v1.88.1 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index a640a1ad51..124c57a830 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.40 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -352,12 +352,14 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | | [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [cluster\_version](#output\_cluster\_version) | The Kubernetes version for the cluster | diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index f57a972247..26f24bd10c 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -29,7 +29,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.40 | ## Providers diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index e62b7a29b4..2f828ec8e4 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -357,9 +357,10 @@ module "disabled_eks" { module "eks_managed_node_group" { source = "../../modules/eks-managed-node-group" - name = "separate-eks-mng" - cluster_name = module.eks.cluster_name - cluster_version = module.eks.cluster_version + name = "separate-eks-mng" + cluster_name = module.eks.cluster_name + cluster_ip_family = module.eks.cluster_ip_family + cluster_service_cidr = module.eks.cluster_service_cidr subnet_ids = module.vpc.private_subnets cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index 876adbdfa4..6f83215f50 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 6f33b44cd2..59b4b981d7 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -19,7 +19,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.40 | ## Providers diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index 876adbdfa4..6f83215f50 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 97d5f963bf..69d70942cb 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -43,7 +43,9 @@ kubectl delete node -l karpenter.sh/provisioner-name=default ```bash # Necessary to avoid removing Terraform's permissions too soon before its finished # cleaning up the resources it deployed inside the cluster -terraform state rm 'module.eks.aws_eks_access_entry.this["cluster_creator_admin"]' || true +terraform state rm 'module.eks.aws_eks_access_entry.this["cluster_creator"]' || true +terraform state rm 'module.eks.aws_eks_access_policy_association.this["cluster_creator_admin"]' || true + terraform destroy ``` @@ -54,7 +56,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.40 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index e48f04af06..9247d6e1ee 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -169,7 +169,7 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "v0.34.0" + version = "0.35.1" wait = false values = [ diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 9cdb8122b9..a43b64f4c4 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { diff --git a/examples/outposts/README.md b/examples/outposts/README.md index e1daea0b83..6558d6b8be 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -48,7 +48,7 @@ terraform destroy | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.40 | | [kubernetes](#requirement\_kubernetes) | >= 2.20 | diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf index 876adbdfa4..6f83215f50 100644 --- a/examples/outposts/prerequisites/versions.tf +++ b/examples/outposts/prerequisites/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf index 6ff0747487..2ac7910678 100644 --- a/examples/outposts/versions.tf +++ b/examples/outposts/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 7aea278b77..1307f1872f 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -25,7 +25,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.40 | ## Providers diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index a9f228e117..c18f29e0dc 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -309,6 +309,9 @@ module "disabled_self_managed_node_group" { source = "../../modules/self-managed-node-group" create = false + + # Hard requirement + cluster_service_cidr = "" } ################################################################################ diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index 876adbdfa4..6f83215f50 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { diff --git a/examples/user_data/README.md b/examples/user_data/README.md index a84b75ac5e..de9b419bb4 100644 --- a/examples/user_data/README.md +++ b/examples/user_data/README.md @@ -17,7 +17,7 @@ $ terraform apply | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [local](#requirement\_local) | >= 2.4 | ## Providers @@ -36,7 +36,9 @@ $ terraform apply | [eks\_mng\_al2023\_no\_op](#module\_eks\_mng\_al2023\_no\_op) | ../../modules/_user_data | n/a | | [eks\_mng\_al2\_additional](#module\_eks\_mng\_al2\_additional) | ../../modules/_user_data | n/a | | [eks\_mng\_al2\_custom\_ami](#module\_eks\_mng\_al2\_custom\_ami) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2\_custom\_ami\_ipv6](#module\_eks\_mng\_al2\_custom\_ami\_ipv6) | ../../modules/_user_data | n/a | | [eks\_mng\_al2\_custom\_template](#module\_eks\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2\_disabled](#module\_eks\_mng\_al2\_disabled) | ../../modules/_user_data | n/a | | [eks\_mng\_al2\_no\_op](#module\_eks\_mng\_al2\_no\_op) | ../../modules/_user_data | n/a | | [eks\_mng\_bottlerocket\_additional](#module\_eks\_mng\_bottlerocket\_additional) | ../../modules/_user_data | n/a | | [eks\_mng\_bottlerocket\_custom\_ami](#module\_eks\_mng\_bottlerocket\_custom\_ami) | ../../modules/_user_data | n/a | @@ -50,6 +52,7 @@ $ terraform apply | [self\_mng\_al2023\_custom\_template](#module\_self\_mng\_al2023\_custom\_template) | ../../modules/_user_data | n/a | | [self\_mng\_al2023\_no\_op](#module\_self\_mng\_al2023\_no\_op) | ../../modules/_user_data | n/a | | [self\_mng\_al2\_bootstrap](#module\_self\_mng\_al2\_bootstrap) | ../../modules/_user_data | n/a | +| [self\_mng\_al2\_bootstrap\_ipv6](#module\_self\_mng\_al2\_bootstrap\_ipv6) | ../../modules/_user_data | n/a | | [self\_mng\_al2\_custom\_template](#module\_self\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a | | [self\_mng\_al2\_no\_op](#module\_self\_mng\_al2\_no\_op) | ../../modules/_user_data | n/a | | [self\_mng\_bottlerocket\_bootstrap](#module\_self\_mng\_bottlerocket\_bootstrap) | ../../modules/_user_data | n/a | @@ -69,6 +72,7 @@ $ terraform apply | [local_file.eks_mng_al2023_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_al2_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_al2_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_al2_custom_ami_ipv6](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_al2_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_bottlerocket_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | @@ -83,6 +87,7 @@ $ terraform apply | [local_file.self_mng_al2023_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_al2023_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_al2_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_al2_bootstrap_ipv6](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_al2_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_bottlerocket_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index 35b15302ba..580b3d5981 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -4,6 +4,7 @@ locals { cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" cluster_service_ipv4_cidr = "172.16.0.0/16" + cluster_service_ipv6_cidr = "fdd3:7636:68bc::/108" cluster_service_cidr = "192.168.0.0/16" } @@ -11,13 +12,25 @@ locals { # EKS managed node group - AL2 ################################################################################ +module "eks_mng_al2_disabled" { + source = "../../modules/_user_data" + + create = false +} + module "eks_mng_al2_no_op" { source = "../../modules/_user_data" + + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr } module "eks_mng_al2_additional" { source = "../../modules/_user_data" + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr + pre_bootstrap_user_data = <<-EOT export USE_MAX_PODS=false EOT @@ -26,10 +39,32 @@ module "eks_mng_al2_additional" { module "eks_mng_al2_custom_ami" { source = "../../modules/_user_data" - cluster_name = local.name - cluster_endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 - cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_cidr = local.cluster_service_ipv4_cidr + + enable_bootstrap_user_data = true + + pre_bootstrap_user_data = <<-EOT + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--instance-type t3a.large'" + + post_bootstrap_user_data = <<-EOT + echo "All done" + EOT +} + +module "eks_mng_al2_custom_ami_ipv6" { + source = "../../modules/_user_data" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_ip_family = "ipv6" + cluster_service_cidr = local.cluster_service_ipv6_cidr enable_bootstrap_user_data = true @@ -47,9 +82,10 @@ module "eks_mng_al2_custom_ami" { module "eks_mng_al2_custom_template" { source = "../../modules/_user_data" - cluster_name = local.name - cluster_endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_cidr = local.cluster_service_ipv4_cidr user_data_template_path = "${path.module}/templates/linux_custom.tpl" @@ -73,6 +109,9 @@ module "eks_mng_al2023_no_op" { source = "../../modules/_user_data" platform = "al2023" + + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr } module "eks_mng_al2023_additional" { @@ -80,6 +119,9 @@ module "eks_mng_al2023_additional" { platform = "al2023" + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr + cloudinit_pre_nodeadm = [{ content = <<-EOT --- @@ -106,9 +148,6 @@ module "eks_mng_al2023_custom_ami" { cluster_auth_base64 = local.cluster_auth_base64 cluster_service_cidr = local.cluster_service_cidr - # Should do nothing - cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr - enable_bootstrap_user_data = true cloudinit_pre_nodeadm = [{ @@ -178,15 +217,16 @@ module "eks_mng_bottlerocket_no_op" { source = "../../modules/_user_data" platform = "bottlerocket" + + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr } module "eks_mng_bottlerocket_additional" { source = "../../modules/_user_data" - platform = "bottlerocket" - - # Should do nothing - cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + platform = "bottlerocket" + cluster_service_cidr = local.cluster_service_cidr bootstrap_extra_args = <<-EOT # extra args added @@ -200,9 +240,10 @@ module "eks_mng_bottlerocket_custom_ami" { platform = "bottlerocket" - cluster_name = local.name - cluster_endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_cidr = local.cluster_service_cidr enable_bootstrap_user_data = true @@ -221,6 +262,8 @@ module "eks_mng_bottlerocket_custom_template" { cluster_name = local.name cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr user_data_template_path = "${path.module}/templates/bottlerocket_custom.tpl" @@ -239,6 +282,9 @@ module "eks_mng_windows_no_op" { source = "../../modules/_user_data" platform = "windows" + + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr } module "eks_mng_windows_additional" { @@ -246,8 +292,8 @@ module "eks_mng_windows_additional" { platform = "windows" - # Should do nothing - cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr pre_bootstrap_user_data = <<-EOT [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' @@ -262,6 +308,8 @@ module "eks_mng_windows_custom_ami" { cluster_name = local.name cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr enable_bootstrap_user_data = true @@ -285,6 +333,9 @@ module "eks_mng_windows_custom_template" { cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr + enable_bootstrap_user_data = true user_data_template_path = "${path.module}/templates/windows_custom.tpl" @@ -308,6 +359,9 @@ module "self_mng_al2_no_op" { source = "../../modules/_user_data" is_eks_managed_node_group = false + + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr } module "self_mng_al2_bootstrap" { @@ -316,9 +370,34 @@ module "self_mng_al2_bootstrap" { enable_bootstrap_user_data = true is_eks_managed_node_group = false - cluster_name = local.name - cluster_endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_cidr = local.cluster_service_ipv4_cidr + + pre_bootstrap_user_data = <<-EOT + echo "foo" + export FOO=bar + EOT + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + + post_bootstrap_user_data = <<-EOT + echo "All done" + EOT +} + +module "self_mng_al2_bootstrap_ipv6" { + source = "../../modules/_user_data" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_ip_family = "ipv6" + cluster_service_cidr = local.cluster_service_ipv6_cidr pre_bootstrap_user_data = <<-EOT echo "foo" @@ -338,9 +417,10 @@ module "self_mng_al2_custom_template" { enable_bootstrap_user_data = true is_eks_managed_node_group = false - cluster_name = local.name - cluster_endpoint = local.cluster_endpoint - cluster_auth_base64 = local.cluster_auth_base64 + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_cidr = local.cluster_service_ipv4_cidr user_data_template_path = "${path.module}/templates/linux_custom.tpl" @@ -366,6 +446,9 @@ module "self_mng_al2023_no_op" { platform = "al2023" is_eks_managed_node_group = false + + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr } module "self_mng_al2023_bootstrap" { @@ -452,6 +535,9 @@ module "self_mng_bottlerocket_no_op" { platform = "bottlerocket" is_eks_managed_node_group = false + + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr } module "self_mng_bottlerocket_bootstrap" { @@ -466,8 +552,8 @@ module "self_mng_bottlerocket_bootstrap" { cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 - # Should do nothing - cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr bootstrap_extra_args = <<-EOT # extra args added @@ -488,6 +574,9 @@ module "self_mng_bottlerocket_custom_template" { cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr + user_data_template_path = "${path.module}/templates/bottlerocket_custom.tpl" bootstrap_extra_args = <<-EOT @@ -507,6 +596,9 @@ module "self_mng_windows_no_op" { platform = "windows" is_eks_managed_node_group = false + + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr } module "self_mng_windows_bootstrap" { @@ -521,8 +613,8 @@ module "self_mng_windows_bootstrap" { cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 - # Should do nothing - cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr pre_bootstrap_user_data = <<-EOT [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' @@ -547,6 +639,9 @@ module "self_mng_windows_custom_template" { cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 + # Hard requirement + cluster_service_cidr = local.cluster_service_cidr + user_data_template_path = "${path.module}/templates/windows_custom.tpl" pre_bootstrap_user_data = <<-EOT diff --git a/examples/user_data/outputs.tf b/examples/user_data/outputs.tf index b407ccec2b..dade25ac65 100644 --- a/examples/user_data/outputs.tf +++ b/examples/user_data/outputs.tf @@ -25,6 +25,11 @@ resource "local_file" "eks_mng_al2_custom_ami" { filename = "${path.module}/rendered/al2/eks-mng-custom-ami.sh" } +resource "local_file" "eks_mng_al2_custom_ami_ipv6" { + content = base64decode(module.eks_mng_al2_custom_ami_ipv6.user_data) + filename = "${path.module}/rendered/al2/eks-mng-custom-ami-ipv6.sh" +} + resource "local_file" "eks_mng_al2_custom_template" { content = base64decode(module.eks_mng_al2_custom_template.user_data) filename = "${path.module}/rendered/al2/eks-mng-custom-template.sh" @@ -116,6 +121,11 @@ resource "local_file" "self_mng_al2_bootstrap" { filename = "${path.module}/rendered/al2/self-mng-bootstrap.sh" } +resource "local_file" "self_mng_al2_bootstrap_ipv6" { + content = base64decode(module.self_mng_al2_bootstrap_ipv6.user_data) + filename = "${path.module}/rendered/al2/self-mng-bootstrap-ipv6.sh" +} + resource "local_file" "self_mng_al2_custom_template" { content = base64decode(module.self_mng_al2_custom_template.user_data) filename = "${path.module}/rendered/al2/self-mng-custom-template.sh" diff --git a/examples/user_data/rendered/al2/eks-mng-custom-ami-ipv6.sh b/examples/user_data/rendered/al2/eks-mng-custom-ami-ipv6.sh new file mode 100755 index 0000000000..fceb7e3571 --- /dev/null +++ b/examples/user_data/rendered/al2/eks-mng-custom-ami-ipv6.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e +export FOO=bar +B64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== +API_SERVER_URL=https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com +/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--instance-type t3a.large' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL \ + --ip-family ipv6 --service-ipv6-cidr fdd3:7636:68bc::/108 +echo "All done" diff --git a/examples/user_data/rendered/al2/eks-mng-custom-ami.sh b/examples/user_data/rendered/al2/eks-mng-custom-ami.sh index 4be9d969b4..c7d92a7ce4 100755 --- a/examples/user_data/rendered/al2/eks-mng-custom-ami.sh +++ b/examples/user_data/rendered/al2/eks-mng-custom-ami.sh @@ -1,8 +1,8 @@ #!/bin/bash set -e export FOO=bar -export SERVICE_IPV4_CIDR=172.16.0.0/16 B64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== API_SERVER_URL=https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com -/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--instance-type t3a.large' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--instance-type t3a.large' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL \ + --ip-family ipv4 --service-ipv4-cidr 172.16.0.0/16 echo "All done" diff --git a/examples/user_data/rendered/al2/eks-mng-custom-template.sh b/examples/user_data/rendered/al2/eks-mng-custom-template.sh index d27a40a135..e18460fa1d 100755 --- a/examples/user_data/rendered/al2/eks-mng-custom-template.sh +++ b/examples/user_data/rendered/al2/eks-mng-custom-template.sh @@ -7,5 +7,6 @@ export FOO=bar # Custom user data template provided for rendering B64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== API_SERVER_URL=https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com -/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL \ + --ip-family ipv4 --service-ipv4-cidr 172.16.0.0/16 echo "All done" diff --git a/examples/user_data/rendered/al2/self-mng-bootstrap-ipv6.sh b/examples/user_data/rendered/al2/self-mng-bootstrap-ipv6.sh new file mode 100755 index 0000000000..b6fd557a13 --- /dev/null +++ b/examples/user_data/rendered/al2/self-mng-bootstrap-ipv6.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e +echo "foo" +export FOO=bar +B64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== +API_SERVER_URL=https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com +/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL \ + --ip-family ipv6 --service-ipv6-cidr fdd3:7636:68bc::/108 +echo "All done" diff --git a/examples/user_data/rendered/al2/self-mng-bootstrap.sh b/examples/user_data/rendered/al2/self-mng-bootstrap.sh index 5dea4bf032..7fcd81973e 100755 --- a/examples/user_data/rendered/al2/self-mng-bootstrap.sh +++ b/examples/user_data/rendered/al2/self-mng-bootstrap.sh @@ -4,5 +4,6 @@ echo "foo" export FOO=bar B64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== API_SERVER_URL=https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com -/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL \ + --ip-family ipv4 --service-ipv4-cidr 172.16.0.0/16 echo "All done" diff --git a/examples/user_data/rendered/al2/self-mng-custom-template.sh b/examples/user_data/rendered/al2/self-mng-custom-template.sh index d27a40a135..e18460fa1d 100755 --- a/examples/user_data/rendered/al2/self-mng-custom-template.sh +++ b/examples/user_data/rendered/al2/self-mng-custom-template.sh @@ -7,5 +7,6 @@ export FOO=bar # Custom user data template provided for rendering B64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== API_SERVER_URL=https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com -/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +/etc/eks/bootstrap.sh ex-user-data --kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL \ + --ip-family ipv4 --service-ipv4-cidr 172.16.0.0/16 echo "All done" diff --git a/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml b/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml index a1bac20219..5c29a516fd 100755 --- a/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml +++ b/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml @@ -2,6 +2,7 @@ "cluster-name" = "ex-user-data" "api-server" = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" "cluster-certificate" = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" +"cluster-dns-ip" = "192.168.0.10" # extra args added [settings.kernel] lockdown = "integrity" diff --git a/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml b/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml index a1bac20219..5c29a516fd 100755 --- a/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml +++ b/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml @@ -2,6 +2,7 @@ "cluster-name" = "ex-user-data" "api-server" = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" "cluster-certificate" = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" +"cluster-dns-ip" = "192.168.0.10" # extra args added [settings.kernel] lockdown = "integrity" diff --git a/examples/user_data/templates/linux_custom.tpl b/examples/user_data/templates/linux_custom.tpl index bfe21f117a..b3cb73a2ab 100644 --- a/examples/user_data/templates/linux_custom.tpl +++ b/examples/user_data/templates/linux_custom.tpl @@ -6,5 +6,6 @@ ${pre_bootstrap_user_data ~} # Custom user data template provided for rendering B64_CLUSTER_CA=${cluster_auth_base64} API_SERVER_URL=${cluster_endpoint} -/etc/eks/bootstrap.sh ${cluster_name} ${bootstrap_extra_args} --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +/etc/eks/bootstrap.sh ${cluster_name} ${bootstrap_extra_args} --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL \ + --ip-family ${cluster_ip_family} --service-${cluster_ip_family}-cidr ${cluster_service_cidr} ${post_bootstrap_user_data ~} diff --git a/examples/user_data/versions.tf b/examples/user_data/versions.tf index af91653bb0..31969d6c02 100644 --- a/examples/user_data/versions.tf +++ b/examples/user_data/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { local = { diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 541f2f880f..0983b121c7 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -9,14 +9,16 @@ See [`examples/user_data/`](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [cloudinit](#requirement\_cloudinit) | >= 2.0 | +| [null](#requirement\_null) | >= 3.0 | ## Providers | Name | Version | |------|---------| | [cloudinit](#provider\_cloudinit) | >= 2.0 | +| [null](#provider\_null) | >= 3.0 | ## Modules @@ -26,6 +28,7 @@ No modules. | Name | Type | |------|------| +| [null_resource.validate_cluster_service_cidr](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [cloudinit_config.al2023_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | | [cloudinit_config.linux_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | @@ -38,9 +41,10 @@ No modules. | [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | | [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | -| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | +| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | [Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [create](#input\_create) | Determines whether to create user-data or not | `bool` | `true` | no | | [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template | `bool` | `false` | no | | [is\_eks\_managed\_node\_group](#input\_is\_eks\_managed\_node\_group) | Determines whether the user data is used on nodes in an EKS managed node group. Used to determine if user data will be appended or not | `bool` | `true` | no | diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index ad3c4e2488..6e44e9ba44 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -1,3 +1,17 @@ +# The `cluster_service_cidr` is required when `create == true` +# This is a hacky way to make that logic work, otherwise Terraform always wants a value +# and supplying any old value like `""` or `null` is not valid and will silently +# fail to join nodes to the cluster +resource "null_resource" "validate_cluster_service_cidr" { + lifecycle { + precondition { + # The length 6 is currently arbitrary, but it's a safe bet that the CIDR will be longer than that + # The main point is that a value needs to be provided when `create = true` + condition = var.create ? length(local.cluster_service_cidr) > 6 : true + error_message = "`cluster_service_cidr` is required when `create = true`." + } + } +} locals { template_path = { @@ -7,6 +21,8 @@ locals { windows = "${path.module}/../../templates/windows_user_data.tpl" } + cluster_service_cidr = try(coalesce(var.cluster_service_ipv4_cidr, var.cluster_service_cidr), "") + user_data = base64encode(templatefile( coalesce(var.user_data_template_path, local.template_path[var.platform]), { @@ -18,14 +34,15 @@ locals { cluster_endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 - # Required by AL2023 - cluster_service_cidr = var.cluster_service_cidr + cluster_service_cidr = local.cluster_service_cidr + cluster_ip_family = var.cluster_ip_family + # Bottlerocket + cluster_dns_ip = try(cidrhost(local.cluster_service_cidr, 10), "") # Optional - cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr != null ? var.cluster_service_ipv4_cidr : "" - bootstrap_extra_args = var.bootstrap_extra_args - pre_bootstrap_user_data = var.pre_bootstrap_user_data - post_bootstrap_user_data = var.post_bootstrap_user_data + bootstrap_extra_args = var.bootstrap_extra_args + pre_bootstrap_user_data = var.pre_bootstrap_user_data + post_bootstrap_user_data = var.post_bootstrap_user_data } )) diff --git a/modules/_user_data/variables.tf b/modules/_user_data/variables.tf index d3d8082dbd..587485a9ba 100644 --- a/modules/_user_data/variables.tf +++ b/modules/_user_data/variables.tf @@ -40,16 +40,21 @@ variable "cluster_auth_base64" { default = "" } -# Currently only used by AL2023 since it can be IPv4 or IPv6 variable "cluster_service_cidr" { description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" type = string default = "" } -# Not used by AL2023 +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = "ipv4" +} + +# TODO - remove at next breaking change variable "cluster_service_ipv4_cidr" { - description = "The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" + description = "[Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" type = string default = null } diff --git a/modules/_user_data/versions.tf b/modules/_user_data/versions.tf index 00dcffe370..9219addeda 100644 --- a/modules/_user_data/versions.tf +++ b/modules/_user_data/versions.tf @@ -1,10 +1,14 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { cloudinit = { source = "hashicorp/cloudinit" version = ">= 2.0" } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } } } diff --git a/modules/aws-auth/README.md b/modules/aws-auth/README.md index d66aa9fca7..5ba490b7f1 100644 --- a/modules/aws-auth/README.md +++ b/modules/aws-auth/README.md @@ -44,7 +44,7 @@ module "eks" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [kubernetes](#requirement\_kubernetes) | >= 2.20 | ## Providers diff --git a/modules/aws-auth/versions.tf b/modules/aws-auth/versions.tf index 8ddb07fa57..f330045476 100644 --- a/modules/aws-auth/versions.tf +++ b/modules/aws-auth/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { kubernetes = { diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 856b014ce7..dbd033bacc 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -63,7 +63,7 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.40 | ## Providers @@ -114,7 +114,7 @@ module "eks_managed_node_group" { | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | | [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | | [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | -| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | +| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | [Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes version. Defaults to EKS Cluster Kubernetes version | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | | [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 10b7fbe678..3dc56cadc1 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -11,12 +11,11 @@ module "user_data" { create = var.create platform = var.platform - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint - cluster_auth_base64 = var.cluster_auth_base64 - - cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr - cluster_service_cidr = var.cluster_service_cidr + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + cluster_ip_family = var.cluster_ip_family + cluster_service_cidr = try(coalesce(var.cluster_service_cidr, var.cluster_service_ipv4_cidr), "") enable_bootstrap_user_data = var.enable_bootstrap_user_data pre_bootstrap_user_data = var.pre_bootstrap_user_data @@ -468,13 +467,21 @@ resource "aws_eks_node_group" "this" { ################################################################################ locals { + create_iam_role = var.create && var.create_iam_role + iam_role_name = coalesce(var.iam_role_name, "${var.name}-eks-node-group") iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" - cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + + ipv4_cni_policy = { for k, v in { + AmazonEKS_CNI_Policy = "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } + ipv6_cni_policy = { for k, v in { + AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } } data "aws_iam_policy_document" "assume_role_policy" { - count = var.create && var.create_iam_role ? 1 : 0 + count = local.create_iam_role ? 1 : 0 statement { sid = "EKSNodeAssumeRole" @@ -488,7 +495,7 @@ data "aws_iam_policy_document" "assume_role_policy" { } resource "aws_iam_role" "this" { - count = var.create && var.create_iam_role ? 1 : 0 + count = local.create_iam_role ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null @@ -504,19 +511,21 @@ resource "aws_iam_role" "this" { # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group resource "aws_iam_role_policy_attachment" "this" { - for_each = { for k, v in { - AmazonEKSWorkerNodePolicy = "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" - AmazonEC2ContainerRegistryReadOnly = "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" - AmazonEKS_CNI_IPv6_Policy = var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" ? local.cni_policy : "" - AmazonEKS_CNI_Policy = var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" ? local.cni_policy : "" - } : k => v if var.create && var.create_iam_role && v != "" } + for_each = { for k, v in merge( + { + AmazonEKSWorkerNodePolicy = "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" + AmazonEC2ContainerRegistryReadOnly = "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" + }, + local.ipv4_cni_policy, + local.ipv6_cni_policy + ) : k => v if local.create_iam_role } policy_arn = each.value role = aws_iam_role.this[0].name } resource "aws_iam_role_policy_attachment" "additional" { - for_each = { for k, v in var.iam_role_additional_policies : k => v if var.create && var.create_iam_role } + for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_role } policy_arn = each.value role = aws_iam_role.this[0].name diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index e0f8812df7..43f17f9f99 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -44,16 +44,15 @@ variable "cluster_auth_base64" { default = "" } -# Currently only used by AL2023 since it can be IPv4 or IPv6 variable "cluster_service_cidr" { description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" type = string default = "" } -# Not used by AL2023 +# TODO - remove at next breaking change variable "cluster_service_ipv4_cidr" { - description = "The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" + description = "[Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" type = string default = null } diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 876adbdfa4..6f83215f50 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 5c2c882450..0ec6ea0e10 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -28,7 +28,7 @@ module "fargate_profile" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.40 | ## Providers diff --git a/modules/fargate-profile/main.tf b/modules/fargate-profile/main.tf index de9dd2d754..7611e22014 100644 --- a/modules/fargate-profile/main.tf +++ b/modules/fargate-profile/main.tf @@ -2,9 +2,17 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} locals { + create_iam_role = var.create && var.create_iam_role + iam_role_name = coalesce(var.iam_role_name, var.name, "fargate-profile") iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" - cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + + ipv4_cni_policy = { for k, v in { + AmazonEKS_CNI_Policy = "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } + ipv6_cni_policy = { for k, v in { + AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } } ################################################################################ @@ -12,7 +20,7 @@ locals { ################################################################################ data "aws_iam_policy_document" "assume_role_policy" { - count = var.create && var.create_iam_role ? 1 : 0 + count = local.create_iam_role ? 1 : 0 statement { effect = "Allow" @@ -26,7 +34,7 @@ data "aws_iam_policy_document" "assume_role_policy" { } resource "aws_iam_role" "this" { - count = var.create && var.create_iam_role ? 1 : 0 + count = local.create_iam_role ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null @@ -41,17 +49,20 @@ resource "aws_iam_role" "this" { } resource "aws_iam_role_policy_attachment" "this" { - for_each = { for k, v in toset(compact([ - "${local.iam_role_policy_prefix}/AmazonEKSFargatePodExecutionRolePolicy", - var.iam_role_attach_cni_policy ? local.cni_policy : "", - ])) : k => v if var.create && var.create_iam_role } + for_each = { for k, v in merge( + { + AmazonEKSFargatePodExecutionRolePolicy = "${local.iam_role_policy_prefix}/AmazonEKSFargatePodExecutionRolePolicy" + }, + local.ipv4_cni_policy, + local.ipv6_cni_policy + ) : k => v if local.create_iam_role } policy_arn = each.value role = aws_iam_role.this[0].name } resource "aws_iam_role_policy_attachment" "additional" { - for_each = { for k, v in var.iam_role_additional_policies : k => v if var.create && var.create_iam_role } + for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_role } policy_arn = each.value role = aws_iam_role.this[0].name diff --git a/modules/fargate-profile/migrations.tf b/modules/fargate-profile/migrations.tf new file mode 100644 index 0000000000..02494f6893 --- /dev/null +++ b/modules/fargate-profile/migrations.tf @@ -0,0 +1,15 @@ +################################################################################ +# Migrations: v20.8 -> v20.9 +################################################################################ + +# Node IAM role policy attachment +# Commercial partition only - `moved` does now allow multiple moves to same target +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy"] + to = aws_iam_role_policy_attachment.this["AmazonEKSFargatePodExecutionRolePolicy"] +} + +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"] + to = aws_iam_role_policy_attachment.this["AmazonEKS_CNI_Policy"] +} diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 876adbdfa4..6f83215f50 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 9d78640c21..ae39e6ccb3 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -84,7 +84,7 @@ module "karpenter" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.40 | ## Providers diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 6ce53f968b..5d82475b6a 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -530,7 +530,13 @@ locals { node_iam_role_name = coalesce(var.node_iam_role_name, "Karpenter-${var.cluster_name}") node_iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" - cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${local.partition}:iam::${local.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.node_iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + + ipv4_cni_policy = { for k, v in { + AmazonEKS_CNI_Policy = "${local.node_iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + } : k => v if var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } + ipv6_cni_policy = { for k, v in { + AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + } : k => v if var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } } data "aws_iam_policy_document" "node_assume_role" { @@ -565,12 +571,14 @@ resource "aws_iam_role" "node" { # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group resource "aws_iam_role_policy_attachment" "node" { - for_each = { for k, v in { - AmazonEKSWorkerNodePolicy = "${local.node_iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" - AmazonEC2ContainerRegistryReadOnly = "${local.node_iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" - AmazonEKS_CNI_IPv6_Policy = var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" ? local.cni_policy : "" - AmazonEKS_CNI_Policy = var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" ? local.cni_policy : "" - } : k => v if local.create_node_iam_role && v != "" } + for_each = { for k, v in merge( + { + AmazonEKSWorkerNodePolicy = "${local.node_iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" + AmazonEC2ContainerRegistryReadOnly = "${local.node_iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" + }, + local.ipv4_cni_policy, + local.ipv6_cni_policy + ) : k => v if local.create_node_iam_role } policy_arn = each.value role = aws_iam_role.node[0].name diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 876adbdfa4..6f83215f50 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index eb39752b8c..1be32f1370 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -42,7 +42,7 @@ module "self_managed_node_group" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.40 | ## Providers diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index c7e5ea6567..77893588b0 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -27,6 +27,7 @@ module "user_data" { cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 + cluster_ip_family = var.cluster_ip_family cluster_service_cidr = var.cluster_service_cidr enable_bootstrap_user_data = true @@ -41,12 +42,14 @@ module "user_data" { ################################################################################ data "aws_ec2_instance_type" "this" { - count = var.create && var.enable_efa_support && local.instance_type_provided ? 1 : 0 + count = local.enable_efa_support ? 1 : 0 instance_type = var.instance_type } locals { + enable_efa_support = var.create && var.enable_efa_support && local.instance_type_provided + instance_type_provided = var.instance_type != "" num_network_cards = try(data.aws_ec2_instance_type.this[0].maximum_network_cards, 0) @@ -60,7 +63,7 @@ locals { } ] - network_interfaces = var.enable_efa_support && local.instance_type_provided ? local.efa_network_interfaces : var.network_interfaces + network_interfaces = local.enable_efa_support ? local.efa_network_interfaces : var.network_interfaces } ################################################################################ @@ -71,7 +74,7 @@ locals { launch_template_name = coalesce(var.launch_template_name, "${var.name}-node-group") security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) - placement = var.create && var.enable_efa_support ? { group_name = aws_placement_group.this[0].name } : var.placement + placement = local.enable_efa_support ? { group_name = aws_placement_group.this[0].name } : var.placement } resource "aws_launch_template" "this" { @@ -695,7 +698,7 @@ resource "aws_autoscaling_group" "this" { target_group_arns = var.target_group_arns termination_policies = var.termination_policies - vpc_zone_identifier = var.enable_efa_support ? data.aws_subnets.efa[0].ids : var.subnet_ids + vpc_zone_identifier = local.enable_efa_support ? data.aws_subnets.efa[0].ids : var.subnet_ids wait_for_capacity_timeout = var.wait_for_capacity_timeout wait_for_elb_capacity = var.wait_for_elb_capacity @@ -734,13 +737,21 @@ resource "aws_autoscaling_group" "this" { ################################################################################ locals { + create_iam_instance_profile = var.create && var.create_iam_instance_profile + iam_role_name = coalesce(var.iam_role_name, "${var.name}-node-group") iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" - cni_policy = var.cluster_ip_family == "ipv6" ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + + ipv4_cni_policy = { for k, v in { + AmazonEKS_CNI_Policy = "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } + ipv6_cni_policy = { for k, v in { + AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } } data "aws_iam_policy_document" "assume_role_policy" { - count = var.create && var.create_iam_instance_profile ? 1 : 0 + count = local.create_iam_instance_profile ? 1 : 0 statement { sid = "EKSNodeAssumeRole" @@ -754,7 +765,7 @@ data "aws_iam_policy_document" "assume_role_policy" { } resource "aws_iam_role" "this" { - count = var.create && var.create_iam_instance_profile ? 1 : 0 + count = local.create_iam_instance_profile ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null @@ -770,26 +781,28 @@ resource "aws_iam_role" "this" { # Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group resource "aws_iam_role_policy_attachment" "this" { - for_each = { for k, v in { - AmazonEKSWorkerNodePolicy = "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" - AmazonEC2ContainerRegistryReadOnly = "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" - AmazonEKS_CNI_IPv6_Policy = var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" ? local.cni_policy : "" - AmazonEKS_CNI_Policy = var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" ? local.cni_policy : "" - } : k => v if var.create && var.create_iam_instance_profile && v != "" } + for_each = { for k, v in merge( + { + AmazonEKSWorkerNodePolicy = "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" + AmazonEC2ContainerRegistryReadOnly = "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" + }, + local.ipv4_cni_policy, + local.ipv6_cni_policy + ) : k => v if local.create_iam_instance_profile } policy_arn = each.value role = aws_iam_role.this[0].name } resource "aws_iam_role_policy_attachment" "additional" { - for_each = { for k, v in var.iam_role_additional_policies : k => v if var.create && var.create_iam_instance_profile } + for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_instance_profile } policy_arn = each.value role = aws_iam_role.this[0].name } resource "aws_iam_instance_profile" "this" { - count = var.create && var.create_iam_instance_profile ? 1 : 0 + count = local.create_iam_instance_profile ? 1 : 0 role = aws_iam_role.this[0].name @@ -809,7 +822,7 @@ resource "aws_iam_instance_profile" "this" { ################################################################################ resource "aws_placement_group" "this" { - count = var.create && var.enable_efa_support ? 1 : 0 + count = local.enable_efa_support ? 1 : 0 name = "${var.cluster_name}-${var.name}" strategy = "cluster" @@ -828,7 +841,7 @@ resource "aws_placement_group" "this" { # Find the availability zones supported by the instance type data "aws_ec2_instance_type_offerings" "this" { - count = var.create && var.enable_efa_support ? 1 : 0 + count = local.enable_efa_support ? 1 : 0 filter { name = "instance-type" @@ -841,7 +854,7 @@ data "aws_ec2_instance_type_offerings" "this" { # Reverse the lookup to find one of the subnets provided based on the availability # availability zone ID of the queried instance type (supported) data "aws_subnets" "efa" { - count = var.create && var.enable_efa_support ? 1 : 0 + count = local.enable_efa_support ? 1 : 0 filter { name = "subnet-id" diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 7c93702e9f..67599a271f 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -38,13 +38,18 @@ variable "cluster_auth_base64" { default = "" } -# Currently only used by AL2023 since it can be IPv4 or IPv6 variable "cluster_service_cidr" { description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" type = string default = "" } +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = "ipv4" +} + variable "pre_bootstrap_user_data" { description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" type = string @@ -546,12 +551,6 @@ variable "create_iam_instance_profile" { default = true } -variable "cluster_ip_family" { - description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" - type = string - default = "ipv4" -} - variable "iam_instance_profile_arn" { description = "Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false`" type = string diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 876adbdfa4..6f83215f50 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { diff --git a/node_groups.tf b/node_groups.tf index 26d5ed7ec8..f5ec719802 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -17,6 +17,8 @@ locals { min_healthy_percentage = 66 } } + + kubernetes_network_config = try(aws_eks_cluster.this[0].kubernetes_network_config[0], {}) } # This sleep resource is used to provide a timed gap between the cluster creation and the downstream dependencies @@ -33,7 +35,7 @@ resource "time_sleep" "this" { cluster_name = aws_eks_cluster.this[0].name cluster_endpoint = aws_eks_cluster.this[0].endpoint cluster_version = aws_eks_cluster.this[0].version - cluster_service_cidr = var.cluster_ip_family == "ipv6" ? aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv6_cidr : aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv4_cidr + cluster_service_cidr = var.cluster_ip_family == "ipv6" ? try(local.kubernetes_network_config.service_ipv6_cidr, "") : try(local.kubernetes_network_config.service_ipv4_cidr, "") cluster_certificate_authority_data = aws_eks_cluster.this[0].certificate_authority[0].data } @@ -296,9 +298,8 @@ module "eks_managed_node_group" { create = try(each.value.create, true) - cluster_name = time_sleep.this[0].triggers["cluster_name"] - cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"]) - cluster_ip_family = var.cluster_ip_family + cluster_name = time_sleep.this[0].triggers["cluster_name"] + cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"]) # EKS Managed Node Group name = try(each.value.name, each.key) @@ -331,6 +332,7 @@ module "eks_managed_node_group" { cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr + cluster_ip_family = var.cluster_ip_family cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") enable_bootstrap_user_data = try(each.value.enable_bootstrap_user_data, var.eks_managed_node_group_defaults.enable_bootstrap_user_data, false) pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.eks_managed_node_group_defaults.pre_bootstrap_user_data, "") @@ -412,8 +414,7 @@ module "self_managed_node_group" { create = try(each.value.create, true) - cluster_name = time_sleep.this[0].triggers["cluster_name"] - cluster_ip_family = var.cluster_ip_family + cluster_name = time_sleep.this[0].triggers["cluster_name"] # Autoscaling Group create_autoscaling_group = try(each.value.create_autoscaling_group, var.self_managed_node_group_defaults.create_autoscaling_group, true) @@ -466,6 +467,7 @@ module "self_managed_node_group" { cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") + cluster_ip_family = var.cluster_ip_family pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.self_managed_node_group_defaults.pre_bootstrap_user_data, "") post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "") bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "") diff --git a/outputs.tf b/outputs.tf index 7621ae8cfa..4891ecbe8f 100644 --- a/outputs.tf +++ b/outputs.tf @@ -52,6 +52,16 @@ output "cluster_primary_security_group_id" { value = try(aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id, null) } +output "cluster_service_cidr" { + description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" + value = try(aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv4_cidr, aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv6_cidr, null) +} + +output "cluster_ip_family" { + description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" + value = try(aws_eks_cluster.this[0].kubernetes_network_config[0].ip_family, null) +} + ################################################################################ # Access Entry ################################################################################ diff --git a/templates/bottlerocket_user_data.tpl b/templates/bottlerocket_user_data.tpl index 640c801438..810ee86fb2 100644 --- a/templates/bottlerocket_user_data.tpl +++ b/templates/bottlerocket_user_data.tpl @@ -3,5 +3,6 @@ "cluster-name" = "${cluster_name}" "api-server" = "${cluster_endpoint}" "cluster-certificate" = "${cluster_auth_base64}" +"cluster-dns-ip" = "${cluster_dns_ip}" %{ endif ~} ${bootstrap_extra_args ~} diff --git a/templates/linux_user_data.tpl b/templates/linux_user_data.tpl index 14acbd2aff..d75d549ccc 100644 --- a/templates/linux_user_data.tpl +++ b/templates/linux_user_data.tpl @@ -3,12 +3,10 @@ set -e %{ endif ~} ${pre_bootstrap_user_data ~} -%{ if length(cluster_service_ipv4_cidr) > 0 ~} -export SERVICE_IPV4_CIDR=${cluster_service_ipv4_cidr} -%{ endif ~} %{ if enable_bootstrap_user_data ~} B64_CLUSTER_CA=${cluster_auth_base64} API_SERVER_URL=${cluster_endpoint} -/etc/eks/bootstrap.sh ${cluster_name} ${bootstrap_extra_args} --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL +/etc/eks/bootstrap.sh ${cluster_name} ${bootstrap_extra_args} --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL \ + --ip-family ${cluster_ip_family} --service-${cluster_ip_family}-cidr ${cluster_service_cidr} ${post_bootstrap_user_data ~} %{ endif ~} diff --git a/versions.tf b/versions.tf index f759deeef2..d0f347a88a 100644 --- a/versions.tf +++ b/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.3.2" required_providers { aws = { From 1627231af669796669ce83e0a4672a7e6d94a0b3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 12 Mar 2024 14:36:49 +0000 Subject: [PATCH 766/962] chore(release): version 20.8.3 [skip ci] ## [20.8.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.2...v20.8.3) (2024-03-12) ### Bug Fixes * Ensure the correct service CIDR and IP family is used in the rendered user data ([#2963](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2963)) ([aeb9f0c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/aeb9f0c990b259320a6c3e5ff93be3f064bb9238)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 238dc4d58e..5948bc9571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.8.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.2...v20.8.3) (2024-03-12) + + +### Bug Fixes + +* Ensure the correct service CIDR and IP family is used in the rendered user data ([#2963](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2963)) ([aeb9f0c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/aeb9f0c990b259320a6c3e5ff93be3f064bb9238)) + ## [20.8.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.1...v20.8.2) (2024-03-11) From 84effa0e30f64ba2fceb7f89c2a822e92f1ee1ea Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 20 Mar 2024 21:33:00 -0400 Subject: [PATCH 767/962] fix: Pass nodeadm user data variables from root module down to nodegroup sub-modules (#2981) --- .pre-commit-config.yaml | 2 +- modules/eks-managed-node-group/README.md | 2 ++ modules/eks-managed-node-group/main.tf | 3 +++ modules/eks-managed-node-group/variables.tf | 22 ++++++++++++++++++++ modules/self-managed-node-group/README.md | 2 ++ modules/self-managed-node-group/main.tf | 3 +++ modules/self-managed-node-group/variables.tf | 22 ++++++++++++++++++++ node_groups.tf | 4 ++++ 8 files changed, 59 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9eda307037..ade44ffaf5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.88.1 + rev: v1.88.2 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index dbd033bacc..f0000e07bf 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -108,6 +108,8 @@ module "eks_managed_node_group" { | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | | [capacity\_type](#input\_capacity\_type) | Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT` | `string` | `"ON_DEMAND"` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 3dc56cadc1..bc6bbb2ced 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -22,6 +22,9 @@ module "user_data" { post_bootstrap_user_data = var.post_bootstrap_user_data bootstrap_extra_args = var.bootstrap_extra_args user_data_template_path = var.user_data_template_path + + cloudinit_pre_nodeadm = var.cloudinit_pre_nodeadm + cloudinit_post_nodeadm = var.cloudinit_post_nodeadm } ################################################################################ diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 43f17f9f99..f12a3da88b 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -81,6 +81,28 @@ variable "user_data_template_path" { default = "" } +variable "cloudinit_pre_nodeadm" { + description = "Array of cloud-init document parts that are created before the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + +variable "cloudinit_post_nodeadm" { + description = "Array of cloud-init document parts that are created after the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + ################################################################################ # Launch template ################################################################################ diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 1be32f1370..487bc6787c 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -89,6 +89,8 @@ module "self_managed_node_group" { | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [capacity\_rebalance](#input\_capacity\_rebalance) | Indicates whether capacity rebalance is enabled | `bool` | `null` | no | | [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 77893588b0..0755eb24c3 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -35,6 +35,9 @@ module "user_data" { post_bootstrap_user_data = var.post_bootstrap_user_data bootstrap_extra_args = var.bootstrap_extra_args user_data_template_path = var.user_data_template_path + + cloudinit_pre_nodeadm = var.cloudinit_pre_nodeadm + cloudinit_post_nodeadm = var.cloudinit_post_nodeadm } ################################################################################ diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 67599a271f..6d1c635b43 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -74,6 +74,28 @@ variable "user_data_template_path" { default = "" } +variable "cloudinit_pre_nodeadm" { + description = "Array of cloud-init document parts that are created before the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + +variable "cloudinit_post_nodeadm" { + description = "Array of cloud-init document parts that are created after the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + ################################################################################ # Launch template ################################################################################ diff --git a/node_groups.tf b/node_groups.tf index f5ec719802..e2eab87d71 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -339,6 +339,8 @@ module "eks_managed_node_group" { post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.eks_managed_node_group_defaults.post_bootstrap_user_data, "") bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.eks_managed_node_group_defaults.bootstrap_extra_args, "") user_data_template_path = try(each.value.user_data_template_path, var.eks_managed_node_group_defaults.user_data_template_path, "") + cloudinit_pre_nodeadm = try(each.value.cloudinit_pre_nodeadm, var.eks_managed_node_group_defaults.cloudinit_pre_nodeadm, []) + cloudinit_post_nodeadm = try(each.value.cloudinit_post_nodeadm, var.eks_managed_node_group_defaults.cloudinit_post_nodeadm, []) # Launch Template create_launch_template = try(each.value.create_launch_template, var.eks_managed_node_group_defaults.create_launch_template, true) @@ -472,6 +474,8 @@ module "self_managed_node_group" { post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "") bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "") user_data_template_path = try(each.value.user_data_template_path, var.self_managed_node_group_defaults.user_data_template_path, "") + cloudinit_pre_nodeadm = try(each.value.cloudinit_pre_nodeadm, var.self_managed_node_group_defaults.cloudinit_pre_nodeadm, []) + cloudinit_post_nodeadm = try(each.value.cloudinit_post_nodeadm, var.self_managed_node_group_defaults.cloudinit_post_nodeadm, []) # Launch Template create_launch_template = try(each.value.create_launch_template, var.self_managed_node_group_defaults.create_launch_template, true) From 70866e6fb26aa46a876f16567a043a9aaee4ed34 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 21 Mar 2024 01:33:28 +0000 Subject: [PATCH 768/962] chore(release): version 20.8.4 [skip ci] ## [20.8.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.3...v20.8.4) (2024-03-21) ### Bug Fixes * Pass nodeadm user data variables from root module down to nodegroup sub-modules ([#2981](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2981)) ([84effa0](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/84effa0e30f64ba2fceb7f89c2a822e92f1ee1ea)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5948bc9571..6dde71bd42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.8.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.3...v20.8.4) (2024-03-21) + + +### Bug Fixes + +* Pass nodeadm user data variables from root module down to nodegroup sub-modules ([#2981](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2981)) ([84effa0](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/84effa0e30f64ba2fceb7f89c2a822e92f1ee1ea)) + ## [20.8.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.2...v20.8.3) (2024-03-12) From 07be37dc7a7506f685717b63e913fbac7737c998 Mon Sep 17 00:00:00 2001 From: Robbie Blaine <4052340+rblaine95@users.noreply.github.com> Date: Mon, 25 Mar 2024 23:01:59 +0200 Subject: [PATCH 769/962] chore: Fix typo in `nodeadm` examples (#2986) * Fix `nodeadm` example typo * `apiVersion: node.eks.aws/v1alpha` -> `apiVersion: node.eks.aws/v1alpha1` * Revert whitespace changes * `tofu apply` in `examples/user_data` * chore: Fix file extension type for MIME multipart user data * chore: Fix line endings based off this https://github.com/hashicorp/terraform/issues/32910 --------- Co-authored-by: Bryant Biggs --- .gitattributes | 1 + examples/eks_managed_node_group/main.tf | 2 +- examples/self_managed_node_group/main.tf | 2 +- examples/user_data/main.tf | 10 ++--- examples/user_data/outputs.tf | 16 +++---- ...g-additional.sh => eks-mng-additional.txt} | 18 ++++---- ...g-additional.sh => eks-mng-additional.txt} | 20 ++++----- ...ng-bootstrap.sh => eks-mng-custom-ami.txt} | 44 +++++++++---------- ...emplate.sh => eks-mng-custom-template.txt} | 44 +++++++++---------- .../{eks-mng-no-op.sh => eks-mng-no-op.txt} | 0 ...g-custom-ami.sh => self-mng-bootstrap.txt} | 44 +++++++++---------- ...mplate.sh => self-mng-custom-template.txt} | 44 +++++++++---------- .../{self-mng-no-op.sh => self-mng-no-op.txt} | 0 13 files changed, 123 insertions(+), 122 deletions(-) create mode 100644 .gitattributes rename examples/user_data/rendered/al2/{eks-mng-additional.sh => eks-mng-additional.txt} (95%) rename examples/user_data/rendered/al2023/{eks-mng-additional.sh => eks-mng-additional.txt} (89%) rename examples/user_data/rendered/al2023/{self-mng-bootstrap.sh => eks-mng-custom-ami.txt} (96%) rename examples/user_data/rendered/al2023/{self-mng-custom-template.sh => eks-mng-custom-template.txt} (97%) rename examples/user_data/rendered/al2023/{eks-mng-no-op.sh => eks-mng-no-op.txt} (100%) rename examples/user_data/rendered/al2023/{eks-mng-custom-ami.sh => self-mng-bootstrap.txt} (96%) rename examples/user_data/rendered/al2023/{eks-mng-custom-template.sh => self-mng-custom-template.txt} (97%) rename examples/user_data/rendered/al2023/{self-mng-no-op.sh => self-mng-no-op.txt} (100%) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..176a458f94 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 2f828ec8e4..1af1800c8d 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -99,7 +99,7 @@ module "eks" { content_type = "application/node.eks.aws" content = <<-EOT --- - apiVersion: node.eks.aws/v1alpha + apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index c18f29e0dc..3074ca6a47 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -82,7 +82,7 @@ module "eks" { content_type = "application/node.eks.aws" content = <<-EOT --- - apiVersion: node.eks.aws/v1alpha + apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index 580b3d5981..21bc0fbb28 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -125,7 +125,7 @@ module "eks_mng_al2023_additional" { cloudinit_pre_nodeadm = [{ content = <<-EOT --- - apiVersion: node.eks.aws/v1alpha + apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: @@ -153,7 +153,7 @@ module "eks_mng_al2023_custom_ami" { cloudinit_pre_nodeadm = [{ content = <<-EOT --- - apiVersion: node.eks.aws/v1alpha + apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: @@ -189,7 +189,7 @@ module "eks_mng_al2023_custom_template" { cloudinit_pre_nodeadm = [{ content = <<-EOT --- - apiVersion: node.eks.aws/v1alpha + apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: @@ -467,7 +467,7 @@ module "self_mng_al2023_bootstrap" { cloudinit_pre_nodeadm = [{ content = <<-EOT --- - apiVersion: node.eks.aws/v1alpha + apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: @@ -505,7 +505,7 @@ module "self_mng_al2023_custom_template" { cloudinit_pre_nodeadm = [{ content = <<-EOT --- - apiVersion: node.eks.aws/v1alpha + apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: diff --git a/examples/user_data/outputs.tf b/examples/user_data/outputs.tf index dade25ac65..c40f6632d0 100644 --- a/examples/user_data/outputs.tf +++ b/examples/user_data/outputs.tf @@ -17,7 +17,7 @@ resource "local_file" "eks_mng_al2_no_op" { resource "local_file" "eks_mng_al2_additional" { content = base64decode(module.eks_mng_al2_additional.user_data) - filename = "${path.module}/rendered/al2/eks-mng-additional.sh" + filename = "${path.module}/rendered/al2/eks-mng-additional.txt" } resource "local_file" "eks_mng_al2_custom_ami" { @@ -41,22 +41,22 @@ resource "local_file" "eks_mng_al2_custom_template" { resource "local_file" "eks_mng_al2023_no_op" { content = base64decode(module.eks_mng_al2023_no_op.user_data) - filename = "${path.module}/rendered/al2023/eks-mng-no-op.sh" + filename = "${path.module}/rendered/al2023/eks-mng-no-op.txt" } resource "local_file" "eks_mng_al2023_additional" { content = base64decode(module.eks_mng_al2023_additional.user_data) - filename = "${path.module}/rendered/al2023/eks-mng-additional.sh" + filename = "${path.module}/rendered/al2023/eks-mng-additional.txt" } resource "local_file" "eks_mng_al2023_custom_ami" { content = base64decode(module.eks_mng_al2023_custom_ami.user_data) - filename = "${path.module}/rendered/al2023/eks-mng-custom-ami.sh" + filename = "${path.module}/rendered/al2023/eks-mng-custom-ami.txt" } resource "local_file" "eks_mng_al2023_custom_template" { content = base64decode(module.eks_mng_al2023_custom_template.user_data) - filename = "${path.module}/rendered/al2023/eks-mng-custom-template.sh" + filename = "${path.module}/rendered/al2023/eks-mng-custom-template.txt" } ################################################################################ @@ -137,17 +137,17 @@ resource "local_file" "self_mng_al2_custom_template" { resource "local_file" "self_mng_al2023_no_op" { content = base64decode(module.self_mng_al2023_no_op.user_data) - filename = "${path.module}/rendered/al2023/self-mng-no-op.sh" + filename = "${path.module}/rendered/al2023/self-mng-no-op.txt" } resource "local_file" "self_mng_al2023_bootstrap" { content = base64decode(module.self_mng_al2023_bootstrap.user_data) - filename = "${path.module}/rendered/al2023/self-mng-bootstrap.sh" + filename = "${path.module}/rendered/al2023/self-mng-bootstrap.txt" } resource "local_file" "self_mng_al2023_custom_template" { content = base64decode(module.self_mng_al2023_custom_template.user_data) - filename = "${path.module}/rendered/al2023/self-mng-custom-template.sh" + filename = "${path.module}/rendered/al2023/self-mng-custom-template.txt" } ################################################################################ diff --git a/examples/user_data/rendered/al2/eks-mng-additional.sh b/examples/user_data/rendered/al2/eks-mng-additional.txt similarity index 95% rename from examples/user_data/rendered/al2/eks-mng-additional.sh rename to examples/user_data/rendered/al2/eks-mng-additional.txt index 70b3142abb..151f0cba7a 100755 --- a/examples/user_data/rendered/al2/eks-mng-additional.sh +++ b/examples/user_data/rendered/al2/eks-mng-additional.txt @@ -1,11 +1,11 @@ Content-Type: multipart/mixed; boundary="//" -MIME-Version: 1.0 - ---// -Content-Transfer-Encoding: 7bit -Content-Type: text/x-shellscript -Mime-Version: 1.0 - +MIME-Version: 1.0 + +--// +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript +Mime-Version: 1.0 + export USE_MAX_PODS=false - ---//-- + +--//-- diff --git a/examples/user_data/rendered/al2023/eks-mng-additional.sh b/examples/user_data/rendered/al2023/eks-mng-additional.txt similarity index 89% rename from examples/user_data/rendered/al2023/eks-mng-additional.sh rename to examples/user_data/rendered/al2023/eks-mng-additional.txt index d8d3a1fdb4..fe3c75c898 100755 --- a/examples/user_data/rendered/al2023/eks-mng-additional.sh +++ b/examples/user_data/rendered/al2023/eks-mng-additional.txt @@ -1,13 +1,13 @@ Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" -MIME-Version: 1.0 - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: application/node.eks.aws -Mime-Version: 1.0 - +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + --- -apiVersion: node.eks.aws/v1alpha +apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: @@ -15,5 +15,5 @@ spec: shutdownGracePeriod: 30s featureGates: DisableKubeletCloudCredentialProviders: true - ---MIMEBOUNDARY-- + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/self-mng-bootstrap.sh b/examples/user_data/rendered/al2023/eks-mng-custom-ami.txt similarity index 96% rename from examples/user_data/rendered/al2023/self-mng-bootstrap.sh rename to examples/user_data/rendered/al2023/eks-mng-custom-ami.txt index 676edd1df6..46362c2030 100755 --- a/examples/user_data/rendered/al2023/self-mng-bootstrap.sh +++ b/examples/user_data/rendered/al2023/eks-mng-custom-ami.txt @@ -1,13 +1,13 @@ Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" -MIME-Version: 1.0 - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: application/node.eks.aws -Mime-Version: 1.0 - +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + --- -apiVersion: node.eks.aws/v1alpha +apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: @@ -15,12 +15,12 @@ spec: shutdownGracePeriod: 30s featureGates: DisableKubeletCloudCredentialProviders: true - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: application/node.eks.aws -Mime-Version: 1.0 - + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + --- apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -30,12 +30,12 @@ spec: apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== cidr: 192.168.0.0/16 - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: text/x-shellscript; charset="us-ascii" -Mime-Version: 1.0 - + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + echo "All done" - ---MIMEBOUNDARY-- + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/self-mng-custom-template.sh b/examples/user_data/rendered/al2023/eks-mng-custom-template.txt similarity index 97% rename from examples/user_data/rendered/al2023/self-mng-custom-template.sh rename to examples/user_data/rendered/al2023/eks-mng-custom-template.txt index 3fe97563e2..a97e188c83 100755 --- a/examples/user_data/rendered/al2023/self-mng-custom-template.sh +++ b/examples/user_data/rendered/al2023/eks-mng-custom-template.txt @@ -1,13 +1,13 @@ Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" -MIME-Version: 1.0 - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: application/node.eks.aws -Mime-Version: 1.0 - +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + --- -apiVersion: node.eks.aws/v1alpha +apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: @@ -15,12 +15,12 @@ spec: shutdownGracePeriod: 30s featureGates: DisableKubeletCloudCredentialProviders: true - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: application/node.eks.aws -Mime-Version: 1.0 - + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + --- apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -34,12 +34,12 @@ spec: config: | [plugins."io.containerd.grpc.v1.cri".containerd] discard_unpacked_layers = false - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: text/x-shellscript; charset="us-ascii" -Mime-Version: 1.0 - + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + echo "All done" - ---MIMEBOUNDARY-- + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/eks-mng-no-op.sh b/examples/user_data/rendered/al2023/eks-mng-no-op.txt similarity index 100% rename from examples/user_data/rendered/al2023/eks-mng-no-op.sh rename to examples/user_data/rendered/al2023/eks-mng-no-op.txt diff --git a/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh b/examples/user_data/rendered/al2023/self-mng-bootstrap.txt similarity index 96% rename from examples/user_data/rendered/al2023/eks-mng-custom-ami.sh rename to examples/user_data/rendered/al2023/self-mng-bootstrap.txt index 676edd1df6..46362c2030 100755 --- a/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh +++ b/examples/user_data/rendered/al2023/self-mng-bootstrap.txt @@ -1,13 +1,13 @@ Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" -MIME-Version: 1.0 - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: application/node.eks.aws -Mime-Version: 1.0 - +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + --- -apiVersion: node.eks.aws/v1alpha +apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: @@ -15,12 +15,12 @@ spec: shutdownGracePeriod: 30s featureGates: DisableKubeletCloudCredentialProviders: true - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: application/node.eks.aws -Mime-Version: 1.0 - + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + --- apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -30,12 +30,12 @@ spec: apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== cidr: 192.168.0.0/16 - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: text/x-shellscript; charset="us-ascii" -Mime-Version: 1.0 - + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + echo "All done" - ---MIMEBOUNDARY-- + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/eks-mng-custom-template.sh b/examples/user_data/rendered/al2023/self-mng-custom-template.txt similarity index 97% rename from examples/user_data/rendered/al2023/eks-mng-custom-template.sh rename to examples/user_data/rendered/al2023/self-mng-custom-template.txt index 3fe97563e2..a97e188c83 100755 --- a/examples/user_data/rendered/al2023/eks-mng-custom-template.sh +++ b/examples/user_data/rendered/al2023/self-mng-custom-template.txt @@ -1,13 +1,13 @@ Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" -MIME-Version: 1.0 - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: application/node.eks.aws -Mime-Version: 1.0 - +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + --- -apiVersion: node.eks.aws/v1alpha +apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: kubelet: @@ -15,12 +15,12 @@ spec: shutdownGracePeriod: 30s featureGates: DisableKubeletCloudCredentialProviders: true - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: application/node.eks.aws -Mime-Version: 1.0 - + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + --- apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -34,12 +34,12 @@ spec: config: | [plugins."io.containerd.grpc.v1.cri".containerd] discard_unpacked_layers = false - ---MIMEBOUNDARY -Content-Transfer-Encoding: 7bit -Content-Type: text/x-shellscript; charset="us-ascii" -Mime-Version: 1.0 - + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + echo "All done" - ---MIMEBOUNDARY-- + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/self-mng-no-op.sh b/examples/user_data/rendered/al2023/self-mng-no-op.txt similarity index 100% rename from examples/user_data/rendered/al2023/self-mng-no-op.sh rename to examples/user_data/rendered/al2023/self-mng-no-op.txt From e2a39c0f261d776e4e18a650aa9068429c4f5ef4 Mon Sep 17 00:00:00 2001 From: Loren Gordon Date: Mon, 8 Apr 2024 15:45:48 -0700 Subject: [PATCH 770/962] fix: Forces cluster outputs to wait until access entries are complete (#3000) --- examples/karpenter/README.md | 5 ----- examples/outposts/README.md | 3 --- outputs.tf | 20 ++++++++++++++++++++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 69d70942cb..dbcbae7428 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -41,11 +41,6 @@ kubectl delete node -l karpenter.sh/provisioner-name=default 2. Remove the resources created by Terraform ```bash -# Necessary to avoid removing Terraform's permissions too soon before its finished -# cleaning up the resources it deployed inside the cluster -terraform state rm 'module.eks.aws_eks_access_entry.this["cluster_creator"]' || true -terraform state rm 'module.eks.aws_eks_access_policy_association.this["cluster_creator_admin"]' || true - terraform destroy ``` diff --git a/examples/outposts/README.md b/examples/outposts/README.md index 6558d6b8be..c5849471da 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -37,9 +37,6 @@ $ terraform apply Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. ```bash -# Necessary to avoid removing Terraform's permissions too soon before its finished -# cleaning up the resources it deployed inside the clsuter -terraform state rm 'module.eks.aws_eks_access_entry.this["cluster_creator_admin"]' || true terraform destroy ``` diff --git a/outputs.tf b/outputs.tf index 4891ecbe8f..231064c2e8 100644 --- a/outputs.tf +++ b/outputs.tf @@ -5,16 +5,31 @@ output "cluster_arn" { description = "The Amazon Resource Name (ARN) of the cluster" value = try(aws_eks_cluster.this[0].arn, null) + + depends_on = [ + aws_eks_access_entry.this, + aws_eks_access_policy_association.this, + ] } output "cluster_certificate_authority_data" { description = "Base64 encoded certificate data required to communicate with the cluster" value = try(aws_eks_cluster.this[0].certificate_authority[0].data, null) + + depends_on = [ + aws_eks_access_entry.this, + aws_eks_access_policy_association.this, + ] } output "cluster_endpoint" { description = "Endpoint for your Kubernetes API server" value = try(aws_eks_cluster.this[0].endpoint, null) + + depends_on = [ + aws_eks_access_entry.this, + aws_eks_access_policy_association.this, + ] } output "cluster_id" { @@ -25,6 +40,11 @@ output "cluster_id" { output "cluster_name" { description = "The name of the EKS cluster" value = try(aws_eks_cluster.this[0].name, "") + + depends_on = [ + aws_eks_access_entry.this, + aws_eks_access_policy_association.this, + ] } output "cluster_oidc_issuer_url" { From afadb14e44d1cdbd852dbae815be377c4034e82a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 8 Apr 2024 22:46:19 +0000 Subject: [PATCH 771/962] chore(release): version 20.8.5 [skip ci] ## [20.8.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.4...v20.8.5) (2024-04-08) ### Bug Fixes * Forces cluster outputs to wait until access entries are complete ([#3000](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3000)) ([e2a39c0](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e2a39c0f261d776e4e18a650aa9068429c4f5ef4)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dde71bd42..d8123ee855 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.8.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.4...v20.8.5) (2024-04-08) + + +### Bug Fixes + +* Forces cluster outputs to wait until access entries are complete ([#3000](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3000)) ([e2a39c0](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e2a39c0f261d776e4e18a650aa9068429c4f5ef4)) + ## [20.8.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.3...v20.8.4) (2024-03-21) From 74d39187d855932dd976da6180eda42dcfe09873 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 8 May 2024 08:04:19 -0400 Subject: [PATCH 772/962] feat: Propagate `ami_type` to self-managed node group; allow using `ami_type` only (#3030) --- .pre-commit-config.yaml | 6 +- docs/compute_resources.md | 9 +-- docs/user_data.md | 18 +++--- examples/eks_managed_node_group/README.md | 2 + examples/eks_managed_node_group/main.tf | 8 +-- examples/eks_managed_node_group/outputs.tf | 10 ++++ examples/fargate_profile/README.md | 2 + examples/fargate_profile/outputs.tf | 10 ++++ examples/karpenter/README.md | 2 + examples/karpenter/outputs.tf | 10 ++++ examples/outposts/README.md | 2 + examples/outposts/outputs.tf | 10 ++++ examples/self_managed_node_group/README.md | 2 + examples/self_managed_node_group/main.tf | 5 +- examples/self_managed_node_group/outputs.tf | 10 ++++ examples/user_data/main.tf | 42 +++++++------- modules/_user_data/README.md | 10 ++-- modules/_user_data/main.tf | 33 +++++++++-- modules/_user_data/outputs.tf | 7 ++- modules/_user_data/variables.tf | 14 +++-- modules/eks-managed-node-group/README.md | 12 ++-- modules/eks-managed-node-group/main.tf | 1 + modules/eks-managed-node-group/outputs.tf | 4 +- modules/eks-managed-node-group/variables.tf | 10 ++-- modules/self-managed-node-group/README.md | 13 +++-- modules/self-managed-node-group/main.tf | 61 ++++++++++++++++---- modules/self-managed-node-group/outputs.tf | 4 +- modules/self-managed-node-group/variables.tf | 14 +++-- node_groups.tf | 4 +- outputs.tf | 2 +- 30 files changed, 239 insertions(+), 98 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ade44ffaf5..e7f43bc81b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.88.2 + rev: v1.89.1 hooks: - id: terraform_fmt - id: terraform_docs @@ -24,8 +24,10 @@ repos: - '--args=--only=terraform_unused_required_providers' - id: terraform_validate - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer - id: trailing-whitespace + - id: mixed-line-ending + args: [--fix=lf] diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 29fa2efb0f..721c29c1ee 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -31,7 +31,6 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ use_custom_launch_template = false ami_type = "BOTTLEROCKET_x86_64" - platform = "bottlerocket" } } ``` @@ -42,7 +41,6 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ eks_managed_node_groups = { bottlerocket_prepend_userdata = { ami_type = "BOTTLEROCKET_x86_64" - platform = "bottlerocket" bootstrap_extra_args = <<-EOT # extra args added @@ -84,7 +82,7 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ eks_managed_node_groups = { bottlerocket_custom_ami = { ami_id = "ami-0ff61e0bcfc81dc94" - platform = "bottlerocket" + ami_type = "BOTTLEROCKET_x86_64" # use module user data template to bootstrap enable_bootstrap_user_data = true @@ -123,15 +121,15 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com } ``` -2. To use Bottlerocket, specify the `platform` as `bottlerocket` and supply a Bottlerocket OS AMI: +2. To use Bottlerocket, specify the `ami_type` as one of the respective `"BOTTLEROCKET_*" types` and supply a Bottlerocket OS AMI: ```hcl cluster_version = "1.27" self_managed_node_groups = { bottlerocket = { - platform = "bottlerocket" ami_id = data.aws_ami.bottlerocket_ami.id + ami_type = "BOTTLEROCKET_x86_64" } } ``` @@ -177,7 +175,6 @@ For example, the following creates 4 AWS EKS Managed Node Groups: # This overrides the OS used bottlerocket = { ami_type = "BOTTLEROCKET_x86_64" - platform = "bottlerocket" } } ``` diff --git a/docs/user_data.md b/docs/user_data.md index 87a28a2b47..1ddc5d1aa7 100644 --- a/docs/user_data.md +++ b/docs/user_data.md @@ -7,12 +7,12 @@ Users can see the various methods of using and providing user data through the [ - AWS EKS Managed Node Groups - By default, any supplied user data is pre-pended to the user data supplied by the EKS Managed Node Group service - If users supply an `ami_id`, the service no longers supplies user data to bootstrap nodes; users can enable `enable_bootstrap_user_data` and use the module provided user data template, or provide their own user data template - - `bottlerocket` platform user data must be in TOML format - - `windows` platform user data must be in powershell/PS1 script format + - AMI types of `BOTTLEROCKET_*`, user data must be in TOML format + - AMI types of `WINDOWS_*`, user data must be in powershell/PS1 script format - Self Managed Node Groups - - `linux` platform (default) -> the user data template (bash/shell script) provided by the module is used as the default; users are able to provide their own user data template - - `bottlerocket` platform -> the user data template (TOML file) provided by the module is used as the default; users are able to provide their own user data template - - `windows` platform -> the user data template (powershell/PS1 script) provided by the module is used as the default; users are able to provide their own user data template + - `AL2_x86_64` AMI type (default) -> the user data template (bash/shell script) provided by the module is used as the default; users are able to provide their own user data template + - `BOTTLEROCKET_*` AMI types -> the user data template (TOML file) provided by the module is used as the default; users are able to provide their own user data template + - `WINDOWS_*` AMI types -> the user data template (powershell/PS1 script) provided by the module is used as the default; users are able to provide their own user data template The templates provided by the module can be found under the [templates directory](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/templates) @@ -37,7 +37,7 @@ When using an EKS managed node group, users have 2 primary routes for interactin bootstrap_extra_args = "..." post_bootstrap_user_data = "..." ``` - - If the AMI is **NOT** an AWS EKS Optimized AMI derivative, or if users wish to have more control over the user data that is supplied to the node when launched, users have the ability to supply their own user data template that will be rendered instead of the module supplied template. Note - only the variables that are supplied to the `templatefile()` for the respective platform/OS are available for use in the supplied template, otherwise users will need to pre-render/pre-populate the template before supplying the final template to the module for rendering as user data. + - If the AMI is **NOT** an AWS EKS Optimized AMI derivative, or if users wish to have more control over the user data that is supplied to the node when launched, users have the ability to supply their own user data template that will be rendered instead of the module supplied template. Note - only the variables that are supplied to the `templatefile()` for the respective AMI type are available for use in the supplied template, otherwise users will need to pre-render/pre-populate the template before supplying the final template to the module for rendering as user data. - Users can use the following variables to facilitate this process: ```hcl user_data_template_path = "./your/user_data.sh" # user supplied bootstrap user data template @@ -46,12 +46,12 @@ When using an EKS managed node group, users have 2 primary routes for interactin post_bootstrap_user_data = "..." ``` -| ℹ️ When using bottlerocket as the desired platform, since the user data for bottlerocket is TOML, all configurations are merged in the one file supplied as user data. Therefore, `pre_bootstrap_user_data` and `post_bootstrap_user_data` are not valid since the bottlerocket OS handles when various settings are applied. If you wish to supply additional configuration settings when using bottlerocket, supply them via the `bootstrap_extra_args` variable. For the linux platform, `bootstrap_extra_args` are settings that will be supplied to the [AWS EKS Optimized AMI bootstrap script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh#L14) such as kubelet extra args, etc. See the [bottlerocket GitHub repository documentation](https://github.com/bottlerocket-os/bottlerocket#description-of-settings) for more details on what settings can be supplied via the `bootstrap_extra_args` variable. | +| ℹ️ When using bottlerocket, the supplied user data (TOML format) is merged in with the values supplied by EKS. Therefore, `pre_bootstrap_user_data` and `post_bootstrap_user_data` are not valid since the bottlerocket OS handles when various settings are applied. If you wish to supply additional configuration settings when using bottlerocket, supply them via the `bootstrap_extra_args` variable. For the `AL2_*` AMI types, `bootstrap_extra_args` are settings that will be supplied to the [AWS EKS Optimized AMI bootstrap script](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh#L14) such as kubelet extra args, etc. See the [bottlerocket GitHub repository documentation](https://github.com/bottlerocket-os/bottlerocket#description-of-settings) for more details on what settings can be supplied via the `bootstrap_extra_args` variable. | | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ### Self Managed Node Group -Self managed node groups require users to provide the necessary bootstrap user data. Users can elect to use the user data template provided by the module for their platform/OS or provide their own user data template for rendering by the module. +Self managed node groups require users to provide the necessary bootstrap user data. Users can elect to use the user data template provided by the module for their respective AMI type or provide their own user data template for rendering by the module. - If the AMI used is a derivative of the [AWS EKS Optimized AMI ](https://github.com/awslabs/amazon-eks-ami), users can opt in to using a template provided by the module that provides the minimum necessary configuration to bootstrap the node when launched: - Users can use the following variables to facilitate this process: @@ -61,7 +61,7 @@ Self managed node groups require users to provide the necessary bootstrap user d bootstrap_extra_args = "..." post_bootstrap_user_data = "..." ``` - - If the AMI is **NOT** an AWS EKS Optimized AMI derivative, or if users wish to have more control over the user data that is supplied to the node when launched, users have the ability to supply their own user data template that will be rendered instead of the module supplied template. Note - only the variables that are supplied to the `templatefile()` for the respective platform/OS are available for use in the supplied template, otherwise users will need to pre-render/pre-populate the template before supplying the final template to the module for rendering as user data. + - If the AMI is **NOT** an AWS EKS Optimized AMI derivative, or if users wish to have more control over the user data that is supplied to the node when launched, users have the ability to supply their own user data template that will be rendered instead of the module supplied template. Note - only the variables that are supplied to the `templatefile()` for the respective AMI type are available for use in the supplied template, otherwise users will need to pre-render/pre-populate the template before supplying the final template to the module for rendering as user data. - Users can use the following variables to facilitate this process: ```hcl user_data_template_path = "./your/user_data.sh" # user supplied bootstrap user data template diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 26f24bd10c..a6faed88e6 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -83,12 +83,14 @@ No inputs. | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | | [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 1af1800c8d..2f1202f5c5 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -90,7 +90,6 @@ module "eks" { # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh) al2023_nodeadm = { ami_type = "AL2023_x86_64_STANDARD" - platform = "al2023" use_latest_ami_release_version = true @@ -119,13 +118,11 @@ module "eks" { use_custom_launch_template = false ami_type = "BOTTLEROCKET_x86_64" - platform = "bottlerocket" } # Adds to the AWS provided user data bottlerocket_add = { ami_type = "BOTTLEROCKET_x86_64" - platform = "bottlerocket" use_latest_ami_release_version = true @@ -141,7 +138,7 @@ module "eks" { bottlerocket_custom = { # Current bottlerocket AMI ami_id = data.aws_ami.eks_default_bottlerocket.image_id - platform = "bottlerocket" + ami_type = "BOTTLEROCKET_x86_64" # Use module user data template to bootstrap enable_bootstrap_user_data = true @@ -285,8 +282,8 @@ module "eks" { # Can be enabled when appropriate for testing/validation create = false - instance_types = ["trn1n.32xlarge"] ami_type = "AL2_x86_64_GPU" + instance_types = ["trn1n.32xlarge"] enable_efa_support = true pre_bootstrap_user_data = <<-EOT @@ -369,7 +366,6 @@ module "eks_managed_node_group" { ] ami_type = "BOTTLEROCKET_x86_64" - platform = "bottlerocket" # this will get added to what AWS provides bootstrap_extra_args = <<-EOT diff --git a/examples/eks_managed_node_group/outputs.tf b/examples/eks_managed_node_group/outputs.tf index 0a8873f479..24183fd207 100644 --- a/examples/eks_managed_node_group/outputs.tf +++ b/examples/eks_managed_node_group/outputs.tf @@ -47,6 +47,16 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +output "cluster_service_cidr" { + description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" + value = module.eks.cluster_service_cidr +} + +output "cluster_ip_family" { + description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" + value = module.eks.cluster_ip_family +} + ################################################################################ # Access Entry ################################################################################ diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 59b4b981d7..604fd3a8ca 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -64,12 +64,14 @@ No inputs. | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | | [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | diff --git a/examples/fargate_profile/outputs.tf b/examples/fargate_profile/outputs.tf index 0a8873f479..24183fd207 100644 --- a/examples/fargate_profile/outputs.tf +++ b/examples/fargate_profile/outputs.tf @@ -47,6 +47,16 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +output "cluster_service_cidr" { + description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" + value = module.eks.cluster_service_cidr +} + +output "cluster_ip_family" { + description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" + value = module.eks.cluster_ip_family +} + ################################################################################ # Access Entry ################################################################################ diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index dbcbae7428..9ccd42c945 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -105,12 +105,14 @@ No inputs. | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | | [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | diff --git a/examples/karpenter/outputs.tf b/examples/karpenter/outputs.tf index 41fe733f87..de0e2e6a28 100644 --- a/examples/karpenter/outputs.tf +++ b/examples/karpenter/outputs.tf @@ -47,6 +47,16 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +output "cluster_service_cidr" { + description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" + value = module.eks.cluster_service_cidr +} + +output "cluster_ip_family" { + description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" + value = module.eks.cluster_ip_family +} + ################################################################################ # Access Entry ################################################################################ diff --git a/examples/outposts/README.md b/examples/outposts/README.md index c5849471da..a454fd2c90 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -96,12 +96,14 @@ terraform destroy | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | | [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | diff --git a/examples/outposts/outputs.tf b/examples/outposts/outputs.tf index 0a8873f479..24183fd207 100644 --- a/examples/outposts/outputs.tf +++ b/examples/outposts/outputs.tf @@ -47,6 +47,16 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +output "cluster_service_cidr" { + description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" + value = module.eks.cluster_service_cidr +} + +output "cluster_ip_family" { + description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" + value = module.eks.cluster_ip_family +} + ################################################################################ # Access Entry ################################################################################ diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 1307f1872f..f6f063f693 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -75,12 +75,14 @@ No inputs. | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | | [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | | [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | | [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 3074ca6a47..1d0e80b2fd 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -75,7 +75,7 @@ module "eks" { # AL2023 node group utilizing new user data format which utilizes nodeadm # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh) al2023_nodeadm = { - platform = "al2023" + ami_type = "AL2023_x86_64_STANDARD" cloudinit_pre_nodeadm = [ { @@ -99,7 +99,7 @@ module "eks" { bottlerocket = { name = "bottlerocket-self-mng" - platform = "bottlerocket" + ami_type = "BOTTLEROCKET_x86_64" ami_id = data.aws_ami.eks_default_bottlerocket.id instance_type = "m5.large" desired_size = 2 @@ -287,6 +287,7 @@ module "eks" { # Can be enabled when appropriate for testing/validation create = false + ami_type = "AL2_x86_64_GPU" instance_type = "trn1n.32xlarge" enable_efa_support = true diff --git a/examples/self_managed_node_group/outputs.tf b/examples/self_managed_node_group/outputs.tf index 0a8873f479..24183fd207 100644 --- a/examples/self_managed_node_group/outputs.tf +++ b/examples/self_managed_node_group/outputs.tf @@ -47,6 +47,16 @@ output "cluster_primary_security_group_id" { value = module.eks.cluster_primary_security_group_id } +output "cluster_service_cidr" { + description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" + value = module.eks.cluster_service_cidr +} + +output "cluster_ip_family" { + description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" + value = module.eks.cluster_ip_family +} + ################################################################################ # Access Entry ################################################################################ diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index 21bc0fbb28..ccebeeae34 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -108,7 +108,7 @@ module "eks_mng_al2_custom_template" { module "eks_mng_al2023_no_op" { source = "../../modules/_user_data" - platform = "al2023" + ami_type = "AL2023_x86_64_STANDARD" # Hard requirement cluster_service_cidr = local.cluster_service_cidr @@ -117,7 +117,7 @@ module "eks_mng_al2023_no_op" { module "eks_mng_al2023_additional" { source = "../../modules/_user_data" - platform = "al2023" + ami_type = "AL2023_x86_64_STANDARD" # Hard requirement cluster_service_cidr = local.cluster_service_cidr @@ -141,7 +141,7 @@ module "eks_mng_al2023_additional" { module "eks_mng_al2023_custom_ami" { source = "../../modules/_user_data" - platform = "al2023" + ami_type = "AL2023_x86_64_STANDARD" cluster_name = local.name cluster_endpoint = local.cluster_endpoint @@ -176,7 +176,7 @@ module "eks_mng_al2023_custom_ami" { module "eks_mng_al2023_custom_template" { source = "../../modules/_user_data" - platform = "al2023" + ami_type = "AL2023_x86_64_STANDARD" cluster_name = local.name cluster_endpoint = local.cluster_endpoint @@ -216,7 +216,7 @@ module "eks_mng_al2023_custom_template" { module "eks_mng_bottlerocket_no_op" { source = "../../modules/_user_data" - platform = "bottlerocket" + ami_type = "BOTTLEROCKET_x86_64" # Hard requirement cluster_service_cidr = local.cluster_service_cidr @@ -225,7 +225,7 @@ module "eks_mng_bottlerocket_no_op" { module "eks_mng_bottlerocket_additional" { source = "../../modules/_user_data" - platform = "bottlerocket" + ami_type = "BOTTLEROCKET_x86_64" cluster_service_cidr = local.cluster_service_cidr bootstrap_extra_args = <<-EOT @@ -238,7 +238,7 @@ module "eks_mng_bottlerocket_additional" { module "eks_mng_bottlerocket_custom_ami" { source = "../../modules/_user_data" - platform = "bottlerocket" + ami_type = "BOTTLEROCKET_x86_64" cluster_name = local.name cluster_endpoint = local.cluster_endpoint @@ -257,7 +257,7 @@ module "eks_mng_bottlerocket_custom_ami" { module "eks_mng_bottlerocket_custom_template" { source = "../../modules/_user_data" - platform = "bottlerocket" + ami_type = "BOTTLEROCKET_x86_64" cluster_name = local.name cluster_endpoint = local.cluster_endpoint @@ -281,7 +281,7 @@ module "eks_mng_bottlerocket_custom_template" { module "eks_mng_windows_no_op" { source = "../../modules/_user_data" - platform = "windows" + ami_type = "WINDOWS_CORE_2022_x86_64" # Hard requirement cluster_service_cidr = local.cluster_service_cidr @@ -290,7 +290,7 @@ module "eks_mng_windows_no_op" { module "eks_mng_windows_additional" { source = "../../modules/_user_data" - platform = "windows" + ami_type = "WINDOWS_CORE_2022_x86_64" # Hard requirement cluster_service_cidr = local.cluster_service_cidr @@ -303,7 +303,7 @@ module "eks_mng_windows_additional" { module "eks_mng_windows_custom_ami" { source = "../../modules/_user_data" - platform = "windows" + ami_type = "WINDOWS_CORE_2022_x86_64" cluster_name = local.name cluster_endpoint = local.cluster_endpoint @@ -327,7 +327,7 @@ module "eks_mng_windows_custom_ami" { module "eks_mng_windows_custom_template" { source = "../../modules/_user_data" - platform = "windows" + ami_type = "WINDOWS_CORE_2022_x86_64" cluster_name = local.name cluster_endpoint = local.cluster_endpoint @@ -443,7 +443,7 @@ module "self_mng_al2_custom_template" { module "self_mng_al2023_no_op" { source = "../../modules/_user_data" - platform = "al2023" + ami_type = "AL2023_x86_64_STANDARD" is_eks_managed_node_group = false @@ -454,7 +454,7 @@ module "self_mng_al2023_no_op" { module "self_mng_al2023_bootstrap" { source = "../../modules/_user_data" - platform = "al2023" + ami_type = "AL2023_x86_64_STANDARD" enable_bootstrap_user_data = true is_eks_managed_node_group = false @@ -490,7 +490,7 @@ module "self_mng_al2023_bootstrap" { module "self_mng_al2023_custom_template" { source = "../../modules/_user_data" - platform = "al2023" + ami_type = "AL2023_x86_64_STANDARD" enable_bootstrap_user_data = true is_eks_managed_node_group = false @@ -532,7 +532,7 @@ module "self_mng_al2023_custom_template" { module "self_mng_bottlerocket_no_op" { source = "../../modules/_user_data" - platform = "bottlerocket" + ami_type = "BOTTLEROCKET_x86_64" is_eks_managed_node_group = false @@ -543,7 +543,7 @@ module "self_mng_bottlerocket_no_op" { module "self_mng_bottlerocket_bootstrap" { source = "../../modules/_user_data" - platform = "bottlerocket" + ami_type = "BOTTLEROCKET_x86_64" enable_bootstrap_user_data = true is_eks_managed_node_group = false @@ -565,7 +565,7 @@ module "self_mng_bottlerocket_bootstrap" { module "self_mng_bottlerocket_custom_template" { source = "../../modules/_user_data" - platform = "bottlerocket" + ami_type = "BOTTLEROCKET_x86_64" enable_bootstrap_user_data = true is_eks_managed_node_group = false @@ -593,7 +593,7 @@ module "self_mng_bottlerocket_custom_template" { module "self_mng_windows_no_op" { source = "../../modules/_user_data" - platform = "windows" + ami_type = "WINDOWS_CORE_2022_x86_64" is_eks_managed_node_group = false @@ -604,7 +604,7 @@ module "self_mng_windows_no_op" { module "self_mng_windows_bootstrap" { source = "../../modules/_user_data" - platform = "windows" + ami_type = "WINDOWS_CORE_2022_x86_64" enable_bootstrap_user_data = true is_eks_managed_node_group = false @@ -630,7 +630,7 @@ module "self_mng_windows_bootstrap" { module "self_mng_windows_custom_template" { source = "../../modules/_user_data" - platform = "windows" + ami_type = "WINDOWS_CORE_2022_x86_64" enable_bootstrap_user_data = true is_eks_managed_node_group = false diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 0983b121c7..1170d753fa 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -36,7 +36,8 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `null` | no | +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | | [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | @@ -48,14 +49,15 @@ No modules. | [create](#input\_create) | Determines whether to create user-data or not | `bool` | `true` | no | | [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template | `bool` | `false` | no | | [is\_eks\_managed\_node\_group](#input\_is\_eks\_managed\_node\_group) | Determines whether the user data is used on nodes in an EKS managed node group. Used to determine if user data will be appended or not | `bool` | `true` | no | -| [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | `string` | `"linux"` | no | -| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | -| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [platform](#input\_platform) | [DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | `string` | `"linux"` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | | [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | ## Outputs | Name | Description | |------|-------------| +| [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023, or `windows | | [user\_data](#output\_user\_data) | Base64 encoded user data rendered for the provided inputs | diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index 6e44e9ba44..0377ebc78c 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -14,6 +14,27 @@ resource "null_resource" "validate_cluster_service_cidr" { } locals { + # Converts AMI type into user data type that represents the underlying format (bash, toml, PS1, nodeadm) + # TODO - platform will be removed in v21.0 and only `ami_type` will be valid + ami_type_to_user_data_type = { + AL2_x86_64 = "linux" + AL2_x86_64_GPU = "linux" + AL2_ARM_64 = "linux" + BOTTLEROCKET_ARM_64 = "bottlerocket" + BOTTLEROCKET_x86_64 = "bottlerocket" + BOTTLEROCKET_ARM_64_NVIDIA = "bottlerocket" + BOTTLEROCKET_x86_64_NVIDIA = "bottlerocket" + WINDOWS_CORE_2019_x86_64 = "windows" + WINDOWS_FULL_2019_x86_64 = "windows" + WINDOWS_CORE_2022_x86_64 = "windows" + WINDOWS_FULL_2022_x86_64 = "windows" + AL2023_x86_64_STANDARD = "al2023" + AL2023_ARM_64_STANDARD = "al2023" + } + # Try to use `ami_type` first, but fall back to current, default behavior + # TODO - will be removed in v21.0 + user_data_type = try(local.ami_type_to_user_data_type[var.ami_type], var.platform) + template_path = { al2023 = "${path.module}/../../templates/al2023_user_data.tpl" bottlerocket = "${path.module}/../../templates/bottlerocket_user_data.tpl" @@ -24,7 +45,7 @@ locals { cluster_service_cidr = try(coalesce(var.cluster_service_ipv4_cidr, var.cluster_service_cidr), "") user_data = base64encode(templatefile( - coalesce(var.user_data_template_path, local.template_path[var.platform]), + coalesce(var.user_data_template_path, local.template_path[local.user_data_type]), { # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami enable_bootstrap_user_data = var.enable_bootstrap_user_data @@ -46,18 +67,18 @@ locals { } )) - platform = { + user_data_type_to_rendered = { al2023 = { user_data = var.create ? try(data.cloudinit_config.al2023_eks_managed_node_group[0].rendered, local.user_data) : "" } bottlerocket = { - user_data = var.create && var.platform == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? local.user_data : "" + user_data = var.create && local.user_data_type == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? local.user_data : "" } linux = { user_data = var.create ? try(data.cloudinit_config.linux_eks_managed_node_group[0].rendered, local.user_data) : "" } windows = { - user_data = var.create && var.platform == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? local.user_data : "" + user_data = var.create && local.user_data_type == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? local.user_data : "" } } } @@ -70,7 +91,7 @@ locals { # See docs for more details -> https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data data "cloudinit_config" "linux_eks_managed_node_group" { - count = var.create && var.platform == "linux" && var.is_eks_managed_node_group && !var.enable_bootstrap_user_data && var.pre_bootstrap_user_data != "" && var.user_data_template_path == "" ? 1 : 0 + count = var.create && local.user_data_type == "linux" && var.is_eks_managed_node_group && !var.enable_bootstrap_user_data && var.pre_bootstrap_user_data != "" && var.user_data_template_path == "" ? 1 : 0 base64_encode = true gzip = false @@ -101,7 +122,7 @@ locals { } data "cloudinit_config" "al2023_eks_managed_node_group" { - count = var.create && var.platform == "al2023" && length(local.nodeadm_cloudinit) > 0 ? 1 : 0 + count = var.create && local.user_data_type == "al2023" && length(local.nodeadm_cloudinit) > 0 ? 1 : 0 base64_encode = true gzip = false diff --git a/modules/_user_data/outputs.tf b/modules/_user_data/outputs.tf index 075801b233..7bebb3f218 100644 --- a/modules/_user_data/outputs.tf +++ b/modules/_user_data/outputs.tf @@ -1,4 +1,9 @@ output "user_data" { description = "Base64 encoded user data rendered for the provided inputs" - value = try(local.platform[var.platform].user_data, null) + value = try(local.user_data_type_to_rendered[local.user_data_type].user_data, null) +} + +output "platform" { + description = "[DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023, or `windows`" + value = local.user_data_type } diff --git a/modules/_user_data/variables.tf b/modules/_user_data/variables.tf index 587485a9ba..58475dcb40 100644 --- a/modules/_user_data/variables.tf +++ b/modules/_user_data/variables.tf @@ -5,11 +5,17 @@ variable "create" { } variable "platform" { - description = "Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based" + description = "[DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" type = string default = "linux" } +variable "ami_type" { + description = "Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values" + type = string + default = null +} + variable "enable_bootstrap_user_data" { description = "Determines whether the bootstrap configurations are populated within the user data template" type = bool @@ -60,19 +66,19 @@ variable "cluster_service_ipv4_cidr" { } variable "pre_bootstrap_user_data" { - description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string default = "" } variable "post_bootstrap_user_data" { - description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string default = "" } variable "bootstrap_extra_args" { - description = "Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" + description = "Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" type = string default = "" } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index f0000e07bf..9cc5680b84 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -102,10 +102,10 @@ module "eks_managed_node_group" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance. If not supplied, EKS will use its own default image | `string` | `""` | no | -| [ami\_release\_version](#input\_ami\_release\_version) | AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version | `string` | `null` | no | +| [ami\_release\_version](#input\_ami\_release\_version) | The AMI version. Defaults to latest AMI release version for the given Kubernetes version and AMI type | `string` | `null` | no | | [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `null` | no | | [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | -| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | | [capacity\_type](#input\_capacity\_type) | Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT` | `string` | `"ON_DEMAND"` | no | | [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | @@ -166,9 +166,9 @@ module "eks_managed_node_group" { | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | | [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | | [placement\_group\_strategy](#input\_placement\_group\_strategy) | The placement group strategy | `string` | `"cluster"` | no | -| [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket` or `linux` based; `windows` is not supported | `string` | `"linux"` | no | -| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | -| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [platform](#input\_platform) | [DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | `string` | `"linux"` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | | [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet | `map(string)` | `{}` | no | | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | | [remote\_access](#input\_remote\_access) | Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false` | `any` | `{}` | no | @@ -205,5 +205,5 @@ module "eks_managed_node_group" { | [node\_group\_resources](#output\_node\_group\_resources) | List of objects containing information about underlying resources | | [node\_group\_status](#output\_node\_group\_status) | Status of the EKS Node Group | | [node\_group\_taints](#output\_node\_group\_taints) | List of objects containing information about taints applied to the node group | -| [platform](#output\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | +| [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index bc6bbb2ced..dc63a0a7af 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -10,6 +10,7 @@ module "user_data" { create = var.create platform = var.platform + ami_type = var.ami_type cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index a72f27b347..8cab6e2c13 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -94,6 +94,6 @@ output "iam_role_unique_id" { ################################################################################ output "platform" { - description = "Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based" - value = var.platform + description = "[DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" + value = module.user_data.platform } diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index f12a3da88b..344b346ad1 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -11,7 +11,7 @@ variable "tags" { } variable "platform" { - description = "Identifies if the OS platform is `bottlerocket` or `linux` based; `windows` is not supported" + description = "[DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" type = string default = "linux" } @@ -58,19 +58,19 @@ variable "cluster_service_ipv4_cidr" { } variable "pre_bootstrap_user_data" { - description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string default = "" } variable "post_bootstrap_user_data" { - description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string default = "" } variable "bootstrap_extra_args" { - description = "Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" + description = "Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" type = string default = "" } @@ -374,7 +374,7 @@ variable "ami_type" { } variable "ami_release_version" { - description = "AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version" + description = "The AMI version. Defaults to latest AMI release version for the given Kubernetes version and AMI type" type = string default = null } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 487bc6787c..282eb2723a 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -70,12 +70,12 @@ module "self_managed_node_group" { | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | | [aws_placement_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/placement_group) | resource | -| [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | | [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_ssm_parameter.ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | | [aws_subnets.efa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs @@ -83,10 +83,11 @@ module "self_managed_node_group" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance | `string` | `""` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the node group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `"AL2_x86_64"` | no | | [autoscaling\_group\_tags](#input\_autoscaling\_group\_tags) | A map of additional tags to add to the autoscaling group created. Tags are applied to the autoscaling group only and are NOT propagated to instances | `map(string)` | `{}` | no | | [availability\_zones](#input\_availability\_zones) | A list of one or more availability zones for the group. Used for EC2-Classic and default subnets when not specified with `subnet_ids` argument. Conflicts with `subnet_ids` | `list(string)` | `null` | no | | [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | -| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [capacity\_rebalance](#input\_capacity\_rebalance) | Indicates whether capacity rebalance is enabled | `bool` | `null` | no | | [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | | [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | @@ -163,9 +164,9 @@ module "self_managed_node_group" { | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | | [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | | [placement\_group](#input\_placement\_group) | The name of the placement group into which you'll launch your instances, if any | `string` | `null` | no | -| [platform](#input\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | `string` | `"linux"` | no | -| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | -| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket` | `string` | `""` | no | +| [platform](#input\_platform) | [DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | `string` | `"linux"` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | | [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet | `map(string)` | `{}` | no | | [protect\_from\_scale\_in](#input\_protect\_from\_scale\_in) | Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events. | `bool` | `false` | no | | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | @@ -214,6 +215,6 @@ module "self_managed_node_group" { | [launch\_template\_id](#output\_launch\_template\_id) | The ID of the launch template | | [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | | [launch\_template\_name](#output\_launch\_template\_name) | The name of the launch template | -| [platform](#output\_platform) | Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based | +| [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | | [user\_data](#output\_user\_data) | Base64 encoded user data | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 0755eb24c3..d4fada6c34 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -1,16 +1,56 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} -data "aws_ami" "eks_default" { - count = var.create && var.create_launch_template ? 1 : 0 +################################################################################ +# AMI SSM Parameter +################################################################################ - filter { - name = "name" - values = ["amazon-eks-node-${var.cluster_version}-v*"] +locals { + # Just to ensure templating doesn't fail when values are not provided + ssm_cluster_version = var.cluster_version != null ? var.cluster_version : "" + + # TODO - Temporary stopgap for backwards compatibility until v21.0 + ami_type_to_user_data_type = { + AL2_x86_64 = "linux" + AL2_x86_64_GPU = "linux" + AL2_ARM_64 = "linux" + BOTTLEROCKET_ARM_64 = "bottlerocket" + BOTTLEROCKET_x86_64 = "bottlerocket" + BOTTLEROCKET_ARM_64_NVIDIA = "bottlerocket" + BOTTLEROCKET_x86_64_NVIDIA = "bottlerocket" + WINDOWS_CORE_2019_x86_64 = "windows" + WINDOWS_FULL_2019_x86_64 = "windows" + WINDOWS_CORE_2022_x86_64 = "windows" + WINDOWS_FULL_2022_x86_64 = "windows" + AL2023_x86_64_STANDARD = "al2023" + AL2023_ARM_64_STANDARD = "al2023" + } + # Try to use `ami_type` first, but fall back to current, default behavior + # TODO - will be removed in v21.0 + user_data_type = try(local.ami_type_to_user_data_type[var.ami_type], var.platform) + + # Map the AMI type to the respective SSM param path + ami_type_to_ssm_param = { + AL2_x86_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2/recommended/image_id" + AL2_x86_64_GPU = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-gpu/recommended/image_id" + AL2_ARM_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-arm64/recommended/image_id" + BOTTLEROCKET_ARM_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/arm64/latest/image_id" + BOTTLEROCKET_x86_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/x86_64/latest/image_id" + BOTTLEROCKET_ARM_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/arm64/latest/image_id" + BOTTLEROCKET_x86_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/x86_64/latest/image_id" + WINDOWS_CORE_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-${local.ssm_cluster_version}/image_id" + WINDOWS_FULL_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-${local.ssm_cluster_version}/image_id" + WINDOWS_CORE_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-EKS_Optimized-${local.ssm_cluster_version}/image_id" + WINDOWS_FULL_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-${local.ssm_cluster_version}/image_id" + AL2023_x86_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/standard/recommended/image_id" + AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/standard/recommended/image_id" } +} + +data "aws_ssm_parameter" "ami" { + count = var.create ? 1 : 0 - most_recent = true - owners = ["amazon"] + name = local.ami_type_to_ssm_param[var.ami_type] } ################################################################################ @@ -21,7 +61,8 @@ module "user_data" { source = "../_user_data" create = var.create - platform = var.platform + platform = local.user_data_type + ami_type = var.ami_type is_eks_managed_node_group = false cluster_name = var.cluster_name @@ -184,7 +225,7 @@ resource "aws_launch_template" "this" { arn = var.create_iam_instance_profile ? aws_iam_instance_profile.this[0].arn : var.iam_instance_profile_arn } - image_id = coalesce(var.ami_id, data.aws_ami.eks_default[0].image_id) + image_id = coalesce(var.ami_id, nonsensitive(data.aws_ssm_parameter.ami[0].value)) instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior dynamic "instance_market_options" { @@ -879,7 +920,7 @@ resource "aws_eks_access_entry" "this" { cluster_name = var.cluster_name principal_arn = var.create_iam_instance_profile ? aws_iam_role.this[0].arn : var.iam_role_arn - type = var.platform == "windows" ? "EC2_WINDOWS" : "EC2_LINUX" + type = local.user_data_type == "windows" ? "EC2_WINDOWS" : "EC2_LINUX" tags = var.tags } diff --git a/modules/self-managed-node-group/outputs.tf b/modules/self-managed-node-group/outputs.tf index aba5b13233..9607810ac3 100644 --- a/modules/self-managed-node-group/outputs.tf +++ b/modules/self-managed-node-group/outputs.tf @@ -142,8 +142,8 @@ output "autoscaling_group_schedule_arns" { ################################################################################ output "platform" { - description = "Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based" - value = var.platform + description = "[DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" + value = module.user_data.platform } output "image_id" { diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 6d1c635b43..3e5f719929 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -11,7 +11,7 @@ variable "tags" { } variable "platform" { - description = "Identifies if the OS platform is `bottlerocket`, `linux`, or `windows` based" + description = "[DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" type = string default = "linux" } @@ -51,19 +51,19 @@ variable "cluster_ip_family" { } variable "pre_bootstrap_user_data" { - description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string default = "" } variable "post_bootstrap_user_data" { - description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `platform` = `bottlerocket`" + description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string default = "" } variable "bootstrap_extra_args" { - description = "Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" + description = "Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" type = string default = "" } @@ -262,6 +262,12 @@ variable "ami_id" { default = "" } +variable "ami_type" { + description = "Type of Amazon Machine Image (AMI) associated with the node group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values" + type = string + default = "AL2_x86_64" +} + variable "cluster_version" { description = "Kubernetes cluster version - used to lookup default AMI ID if one is not provided" type = string diff --git a/node_groups.tf b/node_groups.tf index e2eab87d71..74a5c56232 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -465,7 +465,9 @@ module "self_managed_node_group" { autoscaling_group_tags = try(each.value.autoscaling_group_tags, var.self_managed_node_group_defaults.autoscaling_group_tags, {}) # User data - platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, "linux") + platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, "linux") + # TODO - update this when `var.platform` is removed in v21.0 + ami_type = try(each.value.ami_type, var.self_managed_node_group_defaults.ami_type, "AL2_x86_64") cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") diff --git a/outputs.tf b/outputs.tf index 231064c2e8..45b68a4a23 100644 --- a/outputs.tf +++ b/outputs.tf @@ -74,7 +74,7 @@ output "cluster_primary_security_group_id" { output "cluster_service_cidr" { description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" - value = try(aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv4_cidr, aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv6_cidr, null) + value = var.cluster_ip_family == "ipv6" ? try(aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv6_cidr, null) : try(aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv4_cidr, null) } output "cluster_ip_family" { From f3299cc63dad4465e5c25012144e264e135315be Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 8 May 2024 12:05:03 +0000 Subject: [PATCH 773/962] chore(release): version 20.9.0 [skip ci] ## [20.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.5...v20.9.0) (2024-05-08) ### Features * Propagate `ami_type` to self-managed node group; allow using `ami_type` only ([#3030](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3030)) ([74d3918](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/74d39187d855932dd976da6180eda42dcfe09873)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8123ee855..15673ec838 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.5...v20.9.0) (2024-05-08) + + +### Features + +* Propagate `ami_type` to self-managed node group; allow using `ami_type` only ([#3030](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3030)) ([74d3918](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/74d39187d855932dd976da6180eda42dcfe09873)) + ## [20.8.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.4...v20.8.5) (2024-04-08) From cfcaf27ac78278916ebf3d51dc64a20fe0d7bf01 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Thu, 9 May 2024 13:57:57 +0200 Subject: [PATCH 774/962] feat: Add support for Pod Identity assocation on Karpenter sub-module (#3031) Co-authored-by: Bryant Biggs --- examples/karpenter/README.md | 43 ++++++++++++++-- examples/karpenter/main.tf | 89 +++++++++++----------------------- modules/karpenter/README.md | 7 +++ modules/karpenter/main.tf | 15 ++++++ modules/karpenter/outputs.tf | 14 ++++++ modules/karpenter/variables.tf | 22 +++++++++ 6 files changed, 127 insertions(+), 63 deletions(-) diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 9ccd42c945..def4eb9f21 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -1,6 +1,6 @@ # Karpenter Example -Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is running on EKS Fargate yet Karpenter is providing compute in the form of EC2 instances. +Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is provisioned on top of an EKS Managed Node Group. ## Usage @@ -22,10 +22,47 @@ aws eks --region eu-west-1 update-kubeconfig --name ex-karpenter kubectl scale deployment inflate --replicas 5 # You can watch Karpenter's controller logs with -kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller +kubectl logs -f -n kube-system -l app.kubernetes.io/name=karpenter -c controller ``` -You should see a new node named `karpenter.sh/provisioner-name/default` eventually come up in the console; this was provisioned by Karpenter in response to the scaled deployment above. +Validate if the Amazon EKS Addons Pods are running in the Managed Node Group and the `inflate` application Pods are running on Karpenter provisioned Nodes. + +```bash +kubectl get nodes -L karpenter.sh/registered +``` + +```text +NAME STATUS ROLES AGE VERSION REGISTERED +ip-10-0-16-155.eu-west-1.compute.internal Ready 100s v1.29.3-eks-ae9a62a true +ip-10-0-3-23.eu-west-1.compute.internal Ready 6m1s v1.29.3-eks-ae9a62a +ip-10-0-41-2.eu-west-1.compute.internal Ready 6m3s v1.29.3-eks-ae9a62a +``` + +```sh +kubectl get pods -A -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName +``` + +```text +NAME NODE +inflate-75d744d4c6-nqwz8 ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-nrqnn ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-sp4dx ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-xqzd9 ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-xr6p5 ip-10-0-16-155.eu-west-1.compute.internal +aws-node-mnn7r ip-10-0-3-23.eu-west-1.compute.internal +aws-node-rkmvm ip-10-0-16-155.eu-west-1.compute.internal +aws-node-s4slh ip-10-0-41-2.eu-west-1.compute.internal +coredns-68bd859788-7rcfq ip-10-0-3-23.eu-west-1.compute.internal +coredns-68bd859788-l78hw ip-10-0-41-2.eu-west-1.compute.internal +eks-pod-identity-agent-gbx8l ip-10-0-41-2.eu-west-1.compute.internal +eks-pod-identity-agent-s7vt7 ip-10-0-16-155.eu-west-1.compute.internal +eks-pod-identity-agent-xwgqw ip-10-0-3-23.eu-west-1.compute.internal +karpenter-79f59bdfdc-9q5ff ip-10-0-41-2.eu-west-1.compute.internal +karpenter-79f59bdfdc-cxvhr ip-10-0-3-23.eu-west-1.compute.internal +kube-proxy-7crbl ip-10-0-41-2.eu-west-1.compute.internal +kube-proxy-jtzds ip-10-0-16-155.eu-west-1.compute.internal +kube-proxy-sm42c ip-10-0-3-23.eu-west-1.compute.internal +``` ### Tear Down & Clean-Up diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 9247d6e1ee..800082ef85 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -41,9 +41,8 @@ data "aws_ecrpublic_authorization_token" "token" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.29" - region = "eu-west-1" + name = "ex-${basename(path.cwd)}" + region = "eu-west-1" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -62,62 +61,42 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_public_access = true + cluster_name = local.name + cluster_version = "1.29" # Gives Terraform identity admin access to cluster which will # allow deploying resources (Karpenter) into the cluster enable_cluster_creator_admin_permissions = true + cluster_endpoint_public_access = true cluster_addons = { - coredns = { - configuration_values = jsonencode({ - computeType = "Fargate" - # Ensure that we fully utilize the minimum amount of resources that are supplied by - # Fargate https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html - # Fargate adds 256 MB to each pod's memory reservation for the required Kubernetes - # components (kubelet, kube-proxy, and containerd). Fargate rounds up to the following - # compute configuration that most closely matches the sum of vCPU and memory requests in - # order to ensure pods always have the resources that they need to run. - resources = { - limits = { - cpu = "0.25" - # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the - # request/limit to ensure we can fit within that task - memory = "256M" - } - requests = { - cpu = "0.25" - # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the - # request/limit to ensure we can fit within that task - memory = "256M" - } - } - }) - } - kube-proxy = {} - vpc-cni = {} + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + vpc-cni = {} } vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets - # Fargate profiles use the cluster primary security group so these are not utilized - create_cluster_security_group = false - create_node_security_group = false - - fargate_profiles = { + eks_managed_node_groups = { karpenter = { - selectors = [ - { namespace = "karpenter" } - ] - } - kube-system = { - selectors = [ - { namespace = "kube-system" } - ] + instance_types = ["m5.large"] + + min_size = 2 + max_size = 3 + desired_size = 2 + + taints = { + # This Taint aims to keep just EKS Addons and Karpenter running on this MNG + # The pods that do not tolerate this taint should run on nodes created by Karpenter + addons = { + key = "CriticalAddonsOnly" + value = "true" + effect = "NO_SCHEDULE" + }, + } } } @@ -138,9 +117,8 @@ module "karpenter" { cluster_name = module.eks.cluster_name - # EKS Fargate currently does not support Pod Identity - enable_irsa = true - irsa_oidc_provider_arn = module.eks.oidc_provider_arn + enable_pod_identity = true + create_pod_identity_association = true # Used to attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { @@ -162,14 +140,13 @@ module "karpenter_disabled" { ################################################################################ resource "helm_release" "karpenter" { - namespace = "karpenter" - create_namespace = true + namespace = "kube-system" name = "karpenter" repository = "oci://public.ecr.aws/karpenter" repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "0.35.1" + version = "0.36.1" wait = false values = [ @@ -178,14 +155,6 @@ resource "helm_release" "karpenter" { clusterName: ${module.eks.cluster_name} clusterEndpoint: ${module.eks.cluster_endpoint} interruptionQueue: ${module.karpenter.queue_name} - serviceAccount: - annotations: - eks.amazonaws.com/role-arn: ${module.karpenter.iam_role_arn} - tolerations: - - key: 'eks.amazonaws.com/compute-type' - operator: Equal - value: fargate - effect: "NoSchedule" EOT ] } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index ae39e6ccb3..ec819ed256 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -8,6 +8,7 @@ Configuration in this directory creates the AWS resources required by Karpenter In the following example, the Karpenter module will create: - An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller +- A Pod Identity association to grant Karpenter controller access provided by the IAM Role - A Node IAM role that Karpenter will use to create an Instance Profile for the nodes to receive IAM permissions - An access entry for the Node IAM role to allow nodes to join the cluster - SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. @@ -104,6 +105,7 @@ No modules. | [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | | [aws_eks_access_entry.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | +| [aws_eks_pod_identity_association.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_pod_identity_association) | resource | | [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_policy.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | @@ -135,6 +137,7 @@ No modules. | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created | `bool` | `true` | no | | [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `false` | no | | [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_pod\_identity\_association](#input\_create\_pod\_identity\_association) | Determines whether to create pod identity association | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to enable support for IAM role for service accounts | `bool` | `false` | no | | [enable\_pod\_identity](#input\_enable\_pod\_identity) | Determines whether to enable support for EKS pod identity | `bool` | `true` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | @@ -153,6 +156,7 @@ No modules. | [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no | | [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` |
[
"karpenter:karpenter"
]
| no | | [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | +| [namespace](#input\_namespace) | Namespace to associate with the Karpenter Pod Identity | `string` | `"kube-system"` | no | | [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [node\_iam\_role\_arn](#input\_node\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [node\_iam\_role\_attach\_cni\_policy](#input\_node\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | @@ -168,6 +172,7 @@ No modules. | [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | | [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no | | [rule\_name\_prefix](#input\_rule\_name\_prefix) | Prefix used for all event bridge rules | `string` | `"Karpenter"` | no | +| [service\_account](#input\_service\_account) | Service account to associate with the Karpenter Pod Identity | `string` | `"karpenter"` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | ## Outputs @@ -182,6 +187,7 @@ No modules. | [instance\_profile\_id](#output\_instance\_profile\_id) | Instance profile's ID | | [instance\_profile\_name](#output\_instance\_profile\_name) | Name of the instance profile | | [instance\_profile\_unique](#output\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | +| [namespace](#output\_namespace) | Namespace associated with the Karpenter Pod Identity | | [node\_access\_entry\_arn](#output\_node\_access\_entry\_arn) | Amazon Resource Name (ARN) of the node Access Entry | | [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | | [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | The name of the node IAM role | @@ -189,4 +195,5 @@ No modules. | [queue\_arn](#output\_queue\_arn) | The ARN of the SQS queue | | [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue | | [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue | +| [service\_account](#output\_service\_account) | Service Account associated with the Karpenter Pod Identity | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 5d82475b6a..8a3c9c1b71 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -411,6 +411,21 @@ resource "aws_iam_role_policy_attachment" "controller_additional" { policy_arn = each.value } +################################################################################ +# Pod Identity Association +################################################################################ + +resource "aws_eks_pod_identity_association" "karpenter" { + count = local.create_iam_role && var.enable_pod_identity && var.create_pod_identity_association ? 1 : 0 + + cluster_name = var.cluster_name + namespace = var.namespace + service_account = var.service_account + role_arn = aws_iam_role.controller[0].arn + + tags = var.tags +} + ################################################################################ # Node Termination Queue ################################################################################ diff --git a/modules/karpenter/outputs.tf b/modules/karpenter/outputs.tf index 164baa142c..a71d47242d 100644 --- a/modules/karpenter/outputs.tf +++ b/modules/karpenter/outputs.tf @@ -96,3 +96,17 @@ output "instance_profile_unique" { description = "Stable and unique string identifying the IAM instance profile" value = try(aws_iam_instance_profile.this[0].unique_id, null) } + +################################################################################ +# Pod Identity +################################################################################ + +output "namespace" { + description = "Namespace associated with the Karpenter Pod Identity" + value = var.namespace +} + +output "service_account" { + description = "Service Account associated with the Karpenter Pod Identity" + value = var.service_account +} diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index fc79b1a413..87238c8389 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -138,6 +138,28 @@ variable "irsa_assume_role_condition_test" { default = "StringEquals" } +################################################################################ +# Pod Identity Association +################################################################################ +# TODO - Change default to `true` at next breaking change +variable "create_pod_identity_association" { + description = "Determines whether to create pod identity association" + type = bool + default = false +} + +variable "namespace" { + description = "Namespace to associate with the Karpenter Pod Identity" + type = string + default = "kube-system" +} + +variable "service_account" { + description = "Service account to associate with the Karpenter Pod Identity" + type = string + default = "karpenter" +} + ################################################################################ # Node Termination Queue ################################################################################ From 92fca6fcf94777c55eeccb398fa546a43b958475 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 9 May 2024 11:58:27 +0000 Subject: [PATCH 775/962] chore(release): version 20.10.0 [skip ci] ## [20.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.9.0...v20.10.0) (2024-05-09) ### Features * Add support for Pod Identity assocation on Karpenter sub-module ([#3031](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3031)) ([cfcaf27](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/cfcaf27ac78278916ebf3d51dc64a20fe0d7bf01)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15673ec838..b378a04439 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.9.0...v20.10.0) (2024-05-09) + + +### Features + +* Add support for Pod Identity assocation on Karpenter sub-module ([#3031](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3031)) ([cfcaf27](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/cfcaf27ac78278916ebf3d51dc64a20fe0d7bf01)) + ## [20.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.8.5...v20.9.0) (2024-05-08) From a070d7b2bd92866b91e0963a0f819eec9839ed03 Mon Sep 17 00:00:00 2001 From: Christopher Boyd <6323077+cpboyd@users.noreply.github.com> Date: Thu, 16 May 2024 17:58:31 -0400 Subject: [PATCH 776/962] feat: Add `SourceArn` condition to Fargate profile trust policy (#3039) feat: add `SourceArn` condition to fargate-profile assume_role_policy --- modules/fargate-profile/README.md | 1 + modules/fargate-profile/main.tf | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 0ec6ea0e10..072c2f2e33 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -52,6 +52,7 @@ No modules. | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs diff --git a/modules/fargate-profile/main.tf b/modules/fargate-profile/main.tf index 7611e22014..1e2cf60024 100644 --- a/modules/fargate-profile/main.tf +++ b/modules/fargate-profile/main.tf @@ -1,5 +1,6 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} +data "aws_region" "current" {} locals { create_iam_role = var.create && var.create_iam_role @@ -30,6 +31,15 @@ data "aws_iam_policy_document" "assume_role_policy" { type = "Service" identifiers = ["eks-fargate-pods.amazonaws.com"] } + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + + values = [ + "arn:${data.aws_partition.current.partition}:eks:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:fargateprofile/${var.cluster_name}/*", + ] + } } } From 15a3248f595dd25c5e883e858322ba91e30ffc82 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 16 May 2024 21:59:02 +0000 Subject: [PATCH 777/962] chore(release): version 20.11.0 [skip ci] ## [20.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.10.0...v20.11.0) (2024-05-16) ### Features * Add `SourceArn` condition to Fargate profile trust policy ([#3039](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3039)) ([a070d7b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a070d7b2bd92866b91e0963a0f819eec9839ed03)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b378a04439..f3a415562f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.10.0...v20.11.0) (2024-05-16) + + +### Features + +* Add `SourceArn` condition to Fargate profile trust policy ([#3039](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3039)) ([a070d7b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a070d7b2bd92866b91e0963a0f819eec9839ed03)) + ## [20.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.9.0...v20.10.0) (2024-05-09) From c65d3085037d9c1c87f4fd3a5be1ca1d732dbf7a Mon Sep 17 00:00:00 2001 From: William Paul Evans Date: Tue, 21 May 2024 15:01:04 -0400 Subject: [PATCH 778/962] fix: Ignore changes to `bootstrap_cluster_creator_admin_permissions` which is disabled by default (#3042) * Ignore changes to bootstrap_cluster_creator_admin_permissions to allow upgrading module and importing existing clusters * Update main.tf Co-authored-by: Igor Brites * fix attribute reffernce --------- Co-authored-by: Igor Brites --- main.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/main.tf b/main.tf index 2f169936a0..dd638f298d 100644 --- a/main.tf +++ b/main.tf @@ -102,6 +102,12 @@ resource "aws_eks_cluster" "this" { aws_cloudwatch_log_group.this, aws_iam_policy.cni_ipv6_policy, ] + + lifecycle { + ignore_changes = [ + access_config["bootstrap_cluster_creator_admin_permissions"] + ] + } } resource "aws_ec2_tag" "cluster_primary_security_group" { From 29c19cca00b0a9517d91cf537da2aff15e313ca8 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 21 May 2024 19:01:34 +0000 Subject: [PATCH 779/962] chore(release): version 20.11.1 [skip ci] ## [20.11.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.11.0...v20.11.1) (2024-05-21) ### Bug Fixes * Ignore changes to `bootstrap_cluster_creator_admin_permissions` which is disabled by default ([#3042](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3042)) ([c65d308](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c65d3085037d9c1c87f4fd3a5be1ca1d732dbf7a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a415562f..132af55d9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.11.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.11.0...v20.11.1) (2024-05-21) + + +### Bug Fixes + +* Ignore changes to `bootstrap_cluster_creator_admin_permissions` which is disabled by default ([#3042](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3042)) ([c65d308](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c65d3085037d9c1c87f4fd3a5be1ca1d732dbf7a)) + ## [20.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.10.0...v20.11.0) (2024-05-16) From f90f15e91e88f7fecd466729b9f577fb6e21618d Mon Sep 17 00:00:00 2001 From: gillcaleb Date: Wed, 22 May 2024 14:35:59 -0400 Subject: [PATCH 780/962] docs: Include ref to addon naming and config documentation (#3043) * docs: include ref to addon naming and config * docs: including addon configuration details in the faq * docs: including addon configuration details in the faq doc * docs: including addon configuration details in the faq * chore: Updates --------- Co-authored-by: Caleb Gill Co-authored-by: Bryant Biggs --- docs/faq.md | 216 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 214 insertions(+), 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 395c8c3130..2abd6695c2 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -4,8 +4,9 @@ - [I received an error: `expect exactly one securityGroup tagged with kubernetes.io/cluster/ ...`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#i-received-an-error-expect-exactly-one-securitygroup-tagged-with-kubernetesioclustername-) - [Why are nodes not being registered?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-nodes-not-being-registered) - [Why are there no changes when a node group's `desired_size` is modified?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#why-are-there-no-changes-when-a-node-groups-desired_size-is-modified) -- [How can I deploy Windows based nodes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-can-i-deploy-windows-based-nodes) - [How do I access compute resource attributes?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#how-do-i-access-compute-resource-attributes) +- [What add-ons are available?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#what-add-ons-are-available) +- [What configuration values are available for an add-on?](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/faq.md#what-configuration-values-are-available-for-an-add-on) ### Setting `disk_size` or `remote_access` does not make any changes @@ -65,7 +66,7 @@ Examples of accessing the attributes of the compute resource(s) created by the r ```hcl eks_managed_role_arns = [for group in module.eks_managed_node_group : group.iam_role_arn] -```` +``` - Self Managed Node Group attributes @@ -78,3 +79,214 @@ self_managed_role_arns = [for group in module.self_managed_node_group : group.ia ```hcl fargate_profile_pod_execution_role_arns = [for group in module.fargate_profile : group.fargate_profile_pod_execution_role_arn] ``` + +### What add-ons are available? + +The available EKS add-ons can be [found here](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html). You can also retrieve the available addons from the API using: + +```sh +aws eks describe-addon-versions --query 'addons[*].addonName' +``` + +### What configuration values are available for an add-on? + +You can retrieve the configuration value schema for a given addon using the following command: + +```sh +aws eks describe-addon-configuration --addon-name --addon-version --query 'configurationSchema' --output text | jq +``` + +For example: + +```sh +aws eks describe-addon-configuration --addon-name coredns --addon-version v1.11.1-eksbuild.8 --query 'configurationSchema' --output text | jq +``` + +Returns (at the time of writing): + +```json +{ + "$ref": "#/definitions/Coredns", + "$schema": "http://json-schema.org/draft-06/schema#", + "definitions": { + "Coredns": { + "additionalProperties": false, + "properties": { + "affinity": { + "default": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.io/os", + "operator": "In", + "values": [ + "linux" + ] + }, + { + "key": "kubernetes.io/arch", + "operator": "In", + "values": [ + "amd64", + "arm64" + ] + } + ] + } + ] + } + }, + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "podAffinityTerm": { + "labelSelector": { + "matchExpressions": [ + { + "key": "k8s-app", + "operator": "In", + "values": [ + "kube-dns" + ] + } + ] + }, + "topologyKey": "kubernetes.io/hostname" + }, + "weight": 100 + } + ] + } + } + }, + "description": "Affinity of the coredns pods", + "type": [ + "object", + "null" + ] + }, + "computeType": { + "type": "string" + }, + "corefile": { + "description": "Entire corefile contents to use with installation", + "type": "string" + }, + "nodeSelector": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "podAnnotations": { + "properties": {}, + "title": "The podAnnotations Schema", + "type": "object" + }, + "podDisruptionBudget": { + "description": "podDisruptionBudget configurations", + "enabled": { + "default": true, + "description": "the option to enable managed PDB", + "type": "boolean" + }, + "maxUnavailable": { + "anyOf": [ + { + "pattern": ".*%$", + "type": "string" + }, + { + "type": "integer" + } + ], + "default": 1, + "description": "minAvailable value for managed PDB, can be either string or integer; if it's string, should end with %" + }, + "minAvailable": { + "anyOf": [ + { + "pattern": ".*%$", + "type": "string" + }, + { + "type": "integer" + } + ], + "description": "maxUnavailable value for managed PDB, can be either string or integer; if it's string, should end with %" + }, + "type": "object" + }, + "podLabels": { + "properties": {}, + "title": "The podLabels Schema", + "type": "object" + }, + "replicaCount": { + "type": "integer" + }, + "resources": { + "$ref": "#/definitions/Resources" + }, + "tolerations": { + "default": [ + { + "key": "CriticalAddonsOnly", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node-role.kubernetes.io/control-plane" + } + ], + "description": "Tolerations of the coredns pod", + "items": { + "type": "object" + }, + "type": "array" + }, + "topologySpreadConstraints": { + "description": "The coredns pod topology spread constraints", + "type": "array" + } + }, + "title": "Coredns", + "type": "object" + }, + "Limits": { + "additionalProperties": false, + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + }, + "title": "Limits", + "type": "object" + }, + "Resources": { + "additionalProperties": false, + "properties": { + "limits": { + "$ref": "#/definitions/Limits" + }, + "requests": { + "$ref": "#/definitions/Limits" + } + }, + "title": "Resources", + "type": "object" + } + } +} +``` + +> [!NOTE] +> The available configuration values will vary between add-on versions, +> typically more configuration values will be added in later versions as functionality is enabled by EKS. From 541dbb29f12bb763a34b32acdaea9cea12d7f543 Mon Sep 17 00:00:00 2001 From: Ramsey McGrath <67603285+ramseymcgrathfd@users.noreply.github.com> Date: Tue, 28 May 2024 14:40:29 -0400 Subject: [PATCH 781/962] feat: Support additional cluster DNS IPs with Bottlerocket based AMIs (#3051) * Support adding additional cluster dns ranges to bottlerocket template * Add example for multiple dns ips * fmt, and tf docs * fix: Use a list by default for cluster-dns-ip --------- Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 2 +- examples/user_data/main.tf | 3 +++ .../rendered/bottlerocket/eks-mng-custom-ami.toml | 2 +- .../rendered/bottlerocket/self-mng-bootstrap.toml | 2 +- modules/_user_data/README.md | 1 + modules/_user_data/main.tf | 4 +++- modules/_user_data/variables.tf | 6 ++++++ modules/self-managed-node-group/README.md | 1 + modules/self-managed-node-group/main.tf | 11 ++++++----- modules/self-managed-node-group/variables.tf | 6 ++++++ templates/bottlerocket_user_data.tpl | 2 +- 11 files changed, 30 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7f43bc81b..b577da75b3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.89.1 + rev: v1.90.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index ccebeeae34..9a55b3cf77 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -244,6 +244,9 @@ module "eks_mng_bottlerocket_custom_ami" { cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 cluster_service_cidr = local.cluster_service_cidr + additional_cluster_dns_ips = [ + "169.254.20.10" + ] enable_bootstrap_user_data = true diff --git a/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml b/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml index 5c29a516fd..38b0c46a0b 100755 --- a/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml +++ b/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml @@ -2,7 +2,7 @@ "cluster-name" = "ex-user-data" "api-server" = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" "cluster-certificate" = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" -"cluster-dns-ip" = "192.168.0.10" +"cluster-dns-ip" = ["192.168.0.10", "169.254.20.10"] # extra args added [settings.kernel] lockdown = "integrity" diff --git a/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml b/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml index 5c29a516fd..76f8b82dcd 100755 --- a/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml +++ b/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml @@ -2,7 +2,7 @@ "cluster-name" = "ex-user-data" "api-server" = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" "cluster-certificate" = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" -"cluster-dns-ip" = "192.168.0.10" +"cluster-dns-ip" = ["192.168.0.10"] # extra args added [settings.kernel] lockdown = "integrity" diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 1170d753fa..e5207d9443 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -36,6 +36,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [additional\_cluster\_dns\_ips](#input\_additional\_cluster\_dns\_ips) | Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*` | `list(string)` | `[]` | no | | [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `null` | no | | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index 0377ebc78c..b695ba6cca 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -43,6 +43,7 @@ locals { } cluster_service_cidr = try(coalesce(var.cluster_service_ipv4_cidr, var.cluster_service_cidr), "") + cluster_dns_ips = flatten(concat([try(cidrhost(local.cluster_service_cidr, 10), "")], var.additional_cluster_dns_ips)) user_data = base64encode(templatefile( coalesce(var.user_data_template_path, local.template_path[local.user_data_type]), @@ -57,8 +58,9 @@ locals { cluster_service_cidr = local.cluster_service_cidr cluster_ip_family = var.cluster_ip_family + # Bottlerocket - cluster_dns_ip = try(cidrhost(local.cluster_service_cidr, 10), "") + cluster_dns_ips = "[${join(", ", formatlist("\"%s\"", local.cluster_dns_ips))}]" # Optional bootstrap_extra_args = var.bootstrap_extra_args diff --git a/modules/_user_data/variables.tf b/modules/_user_data/variables.tf index 58475dcb40..d5a1ef1bd3 100644 --- a/modules/_user_data/variables.tf +++ b/modules/_user_data/variables.tf @@ -58,6 +58,12 @@ variable "cluster_ip_family" { default = "ipv4" } +variable "additional_cluster_dns_ips" { + description = "Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*`" + type = list(string) + default = [] +} + # TODO - remove at next breaking change variable "cluster_service_ipv4_cidr" { description = "[Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 282eb2723a..1c025d54f0 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -82,6 +82,7 @@ module "self_managed_node_group" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [additional\_cluster\_dns\_ips](#input\_additional\_cluster\_dns\_ips) | Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*` | `list(string)` | `[]` | no | | [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance | `string` | `""` | no | | [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the node group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `"AL2_x86_64"` | no | | [autoscaling\_group\_tags](#input\_autoscaling\_group\_tags) | A map of additional tags to add to the autoscaling group created. Tags are applied to the autoscaling group only and are NOT propagated to instances | `map(string)` | `{}` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index d4fada6c34..94844c7be3 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -65,11 +65,12 @@ module "user_data" { ami_type = var.ami_type is_eks_managed_node_group = false - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint - cluster_auth_base64 = var.cluster_auth_base64 - cluster_ip_family = var.cluster_ip_family - cluster_service_cidr = var.cluster_service_cidr + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + cluster_ip_family = var.cluster_ip_family + cluster_service_cidr = var.cluster_service_cidr + additional_cluster_dns_ips = var.additional_cluster_dns_ips enable_bootstrap_user_data = true pre_bootstrap_user_data = var.pre_bootstrap_user_data diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 3e5f719929..f3b264bebc 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -50,6 +50,12 @@ variable "cluster_ip_family" { default = "ipv4" } +variable "additional_cluster_dns_ips" { + description = "Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*`" + type = list(string) + default = [] +} + variable "pre_bootstrap_user_data" { description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string diff --git a/templates/bottlerocket_user_data.tpl b/templates/bottlerocket_user_data.tpl index 810ee86fb2..666d666069 100644 --- a/templates/bottlerocket_user_data.tpl +++ b/templates/bottlerocket_user_data.tpl @@ -3,6 +3,6 @@ "cluster-name" = "${cluster_name}" "api-server" = "${cluster_endpoint}" "cluster-certificate" = "${cluster_auth_base64}" -"cluster-dns-ip" = "${cluster_dns_ip}" +"cluster-dns-ip" = ${cluster_dns_ips} %{ endif ~} ${bootstrap_extra_args ~} From 59024ebd0aeacad2c1a47d7c8dad123042814dec Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 28 May 2024 18:41:00 +0000 Subject: [PATCH 782/962] chore(release): version 20.12.0 [skip ci] ## [20.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.11.1...v20.12.0) (2024-05-28) ### Features * Support additional cluster DNS IPs with Bottlerocket based AMIs ([#3051](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3051)) ([541dbb2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/541dbb29f12bb763a34b32acdaea9cea12d7f543)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 132af55d9b..9efda18987 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.11.1...v20.12.0) (2024-05-28) + + +### Features + +* Support additional cluster DNS IPs with Bottlerocket based AMIs ([#3051](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3051)) ([541dbb2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/541dbb29f12bb763a34b32acdaea9cea12d7f543)) + ## [20.11.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.11.0...v20.11.1) (2024-05-21) From 00f076ada4cd78c5c34b8be6e8eba44b628b629a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 31 May 2024 13:01:16 -0400 Subject: [PATCH 783/962] feat: Starting with `1.30`, do not use the cluster OIDC issuer URL by default in the identity provider config (#3055) fix: Starting with `1.30`, do not use the cluster OIDC issuer URL by default in the identity provider config --- docs/UPGRADE-19.0.md | 8 ++++++-- main.tf | 17 +++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/UPGRADE-19.0.md b/docs/UPGRADE-19.0.md index 5bb74723f4..f626129be1 100644 --- a/docs/UPGRADE-19.0.md +++ b/docs/UPGRADE-19.0.md @@ -364,8 +364,12 @@ EKS managed node groups on `v18.x` by default create a security group that does # OIDC Identity provider cluster_identity_providers = { - sts = { - client_id = "sts.amazonaws.com" + cognito = { + client_id = "702vqsrjicklgb7c5b7b50i1gc" + issuer_url = "https://cognito-idp.us-west-2.amazonaws.com/us-west-2_re1u6bpRA" + username_claim = "email" + groups_claim = "cognito:groups" + groups_prefix = "gid:" } } diff --git a/main.tf b/main.tf index dd638f298d..211179d45b 100644 --- a/main.tf +++ b/main.tf @@ -552,6 +552,14 @@ resource "aws_eks_addon" "before_compute" { # Note - this is different from IRSA ################################################################################ +locals { + # Maintain current behavior for <= 1.29, remove default for >= 1.30 + # `null` will return the latest Kubernetes version from the EKS API, which at time of writing is 1.30 + # https://github.com/kubernetes/kubernetes/pull/123561 + idpc_backwards_compat_version = contains(["1.21", "1.22", "1.23", "1.24", "1.25", "1.26", "1.27", "1.28", "1.29"], coalesce(var.cluster_version, "1.30")) + idpc_issuer_url = local.idpc_backwards_compat_version ? try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, null) : null +} + resource "aws_eks_identity_provider_config" "this" { for_each = { for k, v in var.cluster_identity_providers : k => v if local.create && !local.create_outposts_local_cluster } @@ -562,10 +570,11 @@ resource "aws_eks_identity_provider_config" "this" { groups_claim = lookup(each.value, "groups_claim", null) groups_prefix = lookup(each.value, "groups_prefix", null) identity_provider_config_name = try(each.value.identity_provider_config_name, each.key) - issuer_url = try(each.value.issuer_url, aws_eks_cluster.this[0].identity[0].oidc[0].issuer) - required_claims = lookup(each.value, "required_claims", null) - username_claim = lookup(each.value, "username_claim", null) - username_prefix = lookup(each.value, "username_prefix", null) + # TODO - make argument explicitly required on next breaking change + issuer_url = try(each.value.issuer_url, local.idpc_issuer_url) + required_claims = lookup(each.value, "required_claims", null) + username_claim = lookup(each.value, "username_claim", null) + username_prefix = lookup(each.value, "username_prefix", null) } tags = merge(var.tags, try(each.value.tags, {})) From a7ff103478c40e0e1e58b151444e9e91ab0be07a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 31 May 2024 17:01:45 +0000 Subject: [PATCH 784/962] chore(release): version 20.13.0 [skip ci] ## [20.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.12.0...v20.13.0) (2024-05-31) ### Features * Starting with `1.30`, do not use the cluster OIDC issuer URL by default in the identity provider config ([#3055](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3055)) ([00f076a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/00f076ada4cd78c5c34b8be6e8eba44b628b629a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9efda18987..9677a393a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.12.0...v20.13.0) (2024-05-31) + + +### Features + +* Starting with `1.30`, do not use the cluster OIDC issuer URL by default in the identity provider config ([#3055](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3055)) ([00f076a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/00f076ada4cd78c5c34b8be6e8eba44b628b629a)) + ## [20.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.11.1...v20.12.0) (2024-05-28) From 1e319290445a6eb50b53dfb89c9ae9f2949d38d7 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 4 Jun 2024 07:08:47 -0400 Subject: [PATCH 785/962] fix: Correct syntax for correctly ignoring `bootstrap_cluster_creator_admin_permissions` and not all of `access_config` (#3056) --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 211179d45b..4cb1200327 100644 --- a/main.tf +++ b/main.tf @@ -105,7 +105,7 @@ resource "aws_eks_cluster" "this" { lifecycle { ignore_changes = [ - access_config["bootstrap_cluster_creator_admin_permissions"] + access_config[0].bootstrap_cluster_creator_admin_permissions ] } } From 098c6a86ca716dae74bd98974accc29f66178c43 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 4 Jun 2024 11:09:20 +0000 Subject: [PATCH 786/962] chore(release): version 20.13.1 [skip ci] ## [20.13.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.13.0...v20.13.1) (2024-06-04) ### Bug Fixes * Correct syntax for correctly ignoring `bootstrap_cluster_creator_admin_permissions` and not all of `access_config` ([#3056](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3056)) ([1e31929](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/1e319290445a6eb50b53dfb89c9ae9f2949d38d7)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9677a393a3..eae6462aa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.13.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.13.0...v20.13.1) (2024-06-04) + + +### Bug Fixes + +* Correct syntax for correctly ignoring `bootstrap_cluster_creator_admin_permissions` and not all of `access_config` ([#3056](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3056)) ([1e31929](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/1e319290445a6eb50b53dfb89c9ae9f2949d38d7)) + ## [20.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.12.0...v20.13.0) (2024-05-31) From c32cee1adcc90d74545221d7139c1a4ed7587616 Mon Sep 17 00:00:00 2001 From: Koyomi Araragi Date: Wed, 12 Jun 2024 17:01:53 -0300 Subject: [PATCH 787/962] =?UTF-8?q?docs:=20Improve=20karpenter=20example?= =?UTF-8?q?=20to=20use=20service=20account=20for=20helm=20relea=E2=80=A6?= =?UTF-8?q?=20(#3066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: Improve karpenter example to use service account for helm release name * docs: Improve karpenter example to use service account for helm release name * docs: Improve karpenter example to use service account for helm release name --- examples/karpenter/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 800082ef85..b3db54453e 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -151,6 +151,8 @@ resource "helm_release" "karpenter" { values = [ <<-EOT + serviceAccount: + name: ${module.karpenter.service_account} settings: clusterName: ${module.eks.cluster_name} clusterEndpoint: ${module.eks.cluster_endpoint} From ef657bfcb51296841f14cf514ffefb1066f810ee Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 13 Jun 2024 07:50:26 -0400 Subject: [PATCH 788/962] feat: Require users to supply OS via `ami_type` and not via `platform` which is unable to distinquish between the number of variants supported today (#3068) --- .pre-commit-config.yaml | 2 +- modules/self-managed-node-group/README.md | 2 +- modules/self-managed-node-group/main.tf | 5 ++--- modules/self-managed-node-group/variables.tf | 10 ++++++++-- node_groups.tf | 2 +- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b577da75b3..06efda4811 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.90.0 + rev: v1.91.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 1c025d54f0..5fb9cd3212 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -165,7 +165,7 @@ module "self_managed_node_group" { | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | | [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | | [placement\_group](#input\_placement\_group) | The name of the placement group into which you'll launch your instances, if any | `string` | `null` | no | -| [platform](#input\_platform) | [DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | `string` | `"linux"` | no | +| [platform](#input\_platform) | [DEPRECATED - must use `ami_type` instead. Will be removed in `v21.0`] | `string` | `null` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | | [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | | [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet | `map(string)` | `{}` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 94844c7be3..e980453930 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -25,9 +25,8 @@ locals { AL2023_x86_64_STANDARD = "al2023" AL2023_ARM_64_STANDARD = "al2023" } - # Try to use `ami_type` first, but fall back to current, default behavior - # TODO - will be removed in v21.0 - user_data_type = try(local.ami_type_to_user_data_type[var.ami_type], var.platform) + + user_data_type = local.ami_type_to_user_data_type[var.ami_type] # Map the AMI type to the respective SSM param path ami_type_to_ssm_param = { diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index f3b264bebc..aea1269080 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -10,10 +10,16 @@ variable "tags" { default = {} } +# tflint-ignore: terraform_unused_declarations variable "platform" { - description = "[DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" + description = "[DEPRECATED - must use `ami_type` instead. Will be removed in `v21.0`]" type = string - default = "linux" + default = null + + validation { + condition = var.platform == null + error_message = "`platform` is no longer valid due to the number of OS choices. Please provide an [`ami_type`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-amitype) instead." + } } ################################################################################ diff --git a/node_groups.tf b/node_groups.tf index 74a5c56232..7228931071 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -465,7 +465,7 @@ module "self_managed_node_group" { autoscaling_group_tags = try(each.value.autoscaling_group_tags, var.self_managed_node_group_defaults.autoscaling_group_tags, {}) # User data - platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, "linux") + platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, null) # TODO - update this when `var.platform` is removed in v21.0 ami_type = try(each.value.ami_type, var.self_managed_node_group_defaults.ami_type, "AL2_x86_64") cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") From 73b752a1e365808a7214f064845e958e65c548bd Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 13 Jun 2024 11:50:55 +0000 Subject: [PATCH 789/962] chore(release): version 20.14.0 [skip ci] ## [20.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.13.1...v20.14.0) (2024-06-13) ### Features * Require users to supply OS via `ami_type` and not via `platform` which is unable to distinquish between the number of variants supported today ([#3068](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3068)) ([ef657bf](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ef657bfcb51296841f14cf514ffefb1066f810ee)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eae6462aa9..d0b114602b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.13.1...v20.14.0) (2024-06-13) + + +### Features + +* Require users to supply OS via `ami_type` and not via `platform` which is unable to distinquish between the number of variants supported today ([#3068](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3068)) ([ef657bf](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ef657bfcb51296841f14cf514ffefb1066f810ee)) + ## [20.13.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.13.0...v20.13.1) (2024-06-04) From 323fb759d718ed1f295c7278ef89a2e2f060d4a1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 13 Jun 2024 10:51:40 -0400 Subject: [PATCH 790/962] docs: Move examples that are more like test cases to the new `tests/` directory; add better example configurations (#3069) * chore: Move examples that are more like test cases to the new `tests/` directory * chore: Stash * feat: Add better examples for EKS managed node groups * chore: Add better examples for self-managed node groups * chore: Update docs and correct `nodegroup` to `node group` --- README.md | 61 ++++++++----------- docs/UPGRADE-20.0.md | 20 +++--- examples/README.md | 5 +- examples/eks-managed-node-group/README.md | 23 +++++++ examples/eks-managed-node-group/eks-al2.tf | 34 +++++++++++ examples/eks-managed-node-group/eks-al2023.tf | 52 ++++++++++++++++ .../eks-bottlerocket.tf | 52 ++++++++++++++++ examples/eks-managed-node-group/main.tf | 49 +++++++++++++++ .../outputs.tf} | 0 .../variables.tf | 0 .../versions.tf | 0 examples/karpenter/README.md | 6 +- examples/karpenter/main.tf | 7 ++- examples/outposts/README.md | 8 +-- examples/outposts/main.tf | 2 +- examples/outposts/prerequisites/main.tf | 2 +- examples/self-managed-node-group/README.md | 21 +++++++ examples/self-managed-node-group/eks-al2.tf | 33 ++++++++++ .../self-managed-node-group/eks-al2023.tf | 52 ++++++++++++++++ .../eks-bottlerocket.tf | 52 ++++++++++++++++ examples/self-managed-node-group/main.tf | 49 +++++++++++++++ .../outputs.tf} | 0 .../variables.tf | 0 .../versions.tf | 0 modules/_user_data/main.tf | 2 +- modules/eks-managed-node-group/README.md | 2 +- modules/eks-managed-node-group/variables.tf | 2 +- modules/karpenter/README.md | 4 +- modules/self-managed-node-group/README.md | 2 +- modules/self-managed-node-group/variables.tf | 2 +- .../eks-managed-node-group}/README.md | 18 +----- .../eks-managed-node-group}/main.tf | 2 +- .../eks-managed-node-group}/outputs.tf | 0 .../eks-managed-node-group/variables.tf | 0 .../eks-managed-node-group}/versions.tf | 0 .../fargate-profile}/README.md | 8 +-- .../fargate-profile}/main.tf | 6 +- .../fargate-profile}/outputs.tf | 0 .../fargate-profile/variables.tf | 0 tests/fargate-profile/versions.tf | 10 +++ .../self-managed-node-group}/README.md | 14 +---- .../self-managed-node-group}/main.tf | 2 +- .../self-managed-node-group}/outputs.tf | 0 .../self-managed-node-group/variables.tf | 0 tests/self-managed-node-group/versions.tf | 10 +++ .../user_data => tests/user-data}/README.md | 4 +- .../user_data => tests/user-data}/main.tf | 0 .../user_data => tests/user-data}/outputs.tf | 0 .../rendered/al2/eks-mng-additional.txt | 0 .../rendered/al2/eks-mng-custom-ami-ipv6.sh | 0 .../rendered/al2/eks-mng-custom-ami.sh | 0 .../rendered/al2/eks-mng-custom-template.sh | 0 .../user-data/rendered/al2/eks-mng-no-op.sh | 0 .../rendered/al2/self-mng-bootstrap-ipv6.sh | 0 .../rendered/al2/self-mng-bootstrap.sh | 0 .../rendered/al2/self-mng-custom-template.sh | 0 .../user-data/rendered/al2/self-mng-no-op.sh | 0 .../rendered/al2023/eks-mng-additional.txt | 0 .../rendered/al2023/eks-mng-custom-ami.txt | 0 .../al2023/eks-mng-custom-template.txt | 0 .../rendered/al2023/eks-mng-no-op.txt | 0 .../rendered/al2023/self-mng-bootstrap.txt | 0 .../al2023/self-mng-custom-template.txt | 0 .../rendered/al2023/self-mng-no-op.txt | 0 .../bottlerocket/eks-mng-additional.toml | 0 .../bottlerocket/eks-mng-custom-ami.toml | 0 .../bottlerocket/eks-mng-custom-template.toml | 0 .../rendered/bottlerocket/eks-mng-no-op.toml | 0 .../bottlerocket/self-mng-bootstrap.toml | 0 .../self-mng-custom-template.toml | 0 .../rendered/bottlerocket/self-mng-no-op.toml | 0 .../rendered/windows/eks-mng-additional.ps1 | 0 .../rendered/windows/eks-mng-custom-ami.ps1 | 0 .../windows/eks-mng-custom-template.ps1 | 0 .../rendered/windows/eks-mng-no-op.ps1 | 0 .../rendered/windows/self-mng-bootstrap.ps1 | 0 .../windows/self-mng-custom-template.ps1 | 0 .../rendered/windows/self-mng-no-op.ps1 | 0 .../user-data}/templates/al2023_custom.tpl | 0 .../templates/bottlerocket_custom.tpl | 0 .../user-data}/templates/linux_custom.tpl | 0 .../user-data}/templates/windows_custom.tpl | 0 tests/user-data/variables.tf | 0 .../user_data => tests/user-data}/versions.tf | 0 variables.tf | 2 +- 85 files changed, 509 insertions(+), 109 deletions(-) create mode 100644 examples/eks-managed-node-group/README.md create mode 100644 examples/eks-managed-node-group/eks-al2.tf create mode 100644 examples/eks-managed-node-group/eks-al2023.tf create mode 100644 examples/eks-managed-node-group/eks-bottlerocket.tf create mode 100644 examples/eks-managed-node-group/main.tf rename examples/{eks_managed_node_group/variables.tf => eks-managed-node-group/outputs.tf} (100%) rename examples/{fargate_profile => eks-managed-node-group}/variables.tf (100%) rename examples/{eks_managed_node_group => eks-managed-node-group}/versions.tf (100%) create mode 100644 examples/self-managed-node-group/README.md create mode 100644 examples/self-managed-node-group/eks-al2.tf create mode 100644 examples/self-managed-node-group/eks-al2023.tf create mode 100644 examples/self-managed-node-group/eks-bottlerocket.tf create mode 100644 examples/self-managed-node-group/main.tf rename examples/{self_managed_node_group/variables.tf => self-managed-node-group/outputs.tf} (100%) rename examples/{user_data => self-managed-node-group}/variables.tf (100%) rename examples/{fargate_profile => self-managed-node-group}/versions.tf (100%) rename {examples/eks_managed_node_group => tests/eks-managed-node-group}/README.md (88%) rename {examples/eks_managed_node_group => tests/eks-managed-node-group}/main.tf (99%) rename {examples/eks_managed_node_group => tests/eks-managed-node-group}/outputs.tf (100%) rename examples/user_data/rendered/al2/eks-mng-no-op.sh => tests/eks-managed-node-group/variables.tf (100%) mode change 100755 => 100644 rename {examples/self_managed_node_group => tests/eks-managed-node-group}/versions.tf (100%) rename {examples/fargate_profile => tests/fargate-profile}/README.md (97%) rename {examples/fargate_profile => tests/fargate-profile}/main.tf (96%) rename {examples/fargate_profile => tests/fargate-profile}/outputs.tf (100%) rename examples/user_data/rendered/al2/self-mng-no-op.sh => tests/fargate-profile/variables.tf (100%) mode change 100755 => 100644 create mode 100644 tests/fargate-profile/versions.tf rename {examples/self_managed_node_group => tests/self-managed-node-group}/README.md (90%) rename {examples/self_managed_node_group => tests/self-managed-node-group}/main.tf (99%) rename {examples/self_managed_node_group => tests/self-managed-node-group}/outputs.tf (100%) rename examples/user_data/rendered/al2023/eks-mng-no-op.txt => tests/self-managed-node-group/variables.tf (100%) mode change 100755 => 100644 create mode 100644 tests/self-managed-node-group/versions.tf rename {examples/user_data => tests/user-data}/README.md (99%) rename {examples/user_data => tests/user-data}/main.tf (100%) rename {examples/user_data => tests/user-data}/outputs.tf (100%) rename {examples/user_data => tests/user-data}/rendered/al2/eks-mng-additional.txt (100%) rename {examples/user_data => tests/user-data}/rendered/al2/eks-mng-custom-ami-ipv6.sh (100%) rename {examples/user_data => tests/user-data}/rendered/al2/eks-mng-custom-ami.sh (100%) rename {examples/user_data => tests/user-data}/rendered/al2/eks-mng-custom-template.sh (100%) rename examples/user_data/rendered/al2023/self-mng-no-op.txt => tests/user-data/rendered/al2/eks-mng-no-op.sh (100%) rename {examples/user_data => tests/user-data}/rendered/al2/self-mng-bootstrap-ipv6.sh (100%) rename {examples/user_data => tests/user-data}/rendered/al2/self-mng-bootstrap.sh (100%) rename {examples/user_data => tests/user-data}/rendered/al2/self-mng-custom-template.sh (100%) rename examples/user_data/rendered/bottlerocket/eks-mng-no-op.toml => tests/user-data/rendered/al2/self-mng-no-op.sh (100%) rename {examples/user_data => tests/user-data}/rendered/al2023/eks-mng-additional.txt (100%) rename {examples/user_data => tests/user-data}/rendered/al2023/eks-mng-custom-ami.txt (100%) rename {examples/user_data => tests/user-data}/rendered/al2023/eks-mng-custom-template.txt (100%) rename examples/user_data/rendered/bottlerocket/self-mng-no-op.toml => tests/user-data/rendered/al2023/eks-mng-no-op.txt (100%) rename {examples/user_data => tests/user-data}/rendered/al2023/self-mng-bootstrap.txt (100%) rename {examples/user_data => tests/user-data}/rendered/al2023/self-mng-custom-template.txt (100%) rename examples/user_data/rendered/windows/eks-mng-no-op.ps1 => tests/user-data/rendered/al2023/self-mng-no-op.txt (100%) rename {examples/user_data => tests/user-data}/rendered/bottlerocket/eks-mng-additional.toml (100%) rename {examples/user_data => tests/user-data}/rendered/bottlerocket/eks-mng-custom-ami.toml (100%) rename {examples/user_data => tests/user-data}/rendered/bottlerocket/eks-mng-custom-template.toml (100%) rename examples/user_data/rendered/windows/self-mng-no-op.ps1 => tests/user-data/rendered/bottlerocket/eks-mng-no-op.toml (100%) rename {examples/user_data => tests/user-data}/rendered/bottlerocket/self-mng-bootstrap.toml (100%) rename {examples/user_data => tests/user-data}/rendered/bottlerocket/self-mng-custom-template.toml (100%) create mode 100755 tests/user-data/rendered/bottlerocket/self-mng-no-op.toml rename {examples/user_data => tests/user-data}/rendered/windows/eks-mng-additional.ps1 (100%) rename {examples/user_data => tests/user-data}/rendered/windows/eks-mng-custom-ami.ps1 (100%) rename {examples/user_data => tests/user-data}/rendered/windows/eks-mng-custom-template.ps1 (100%) create mode 100755 tests/user-data/rendered/windows/eks-mng-no-op.ps1 rename {examples/user_data => tests/user-data}/rendered/windows/self-mng-bootstrap.ps1 (100%) rename {examples/user_data => tests/user-data}/rendered/windows/self-mng-custom-template.ps1 (100%) create mode 100755 tests/user-data/rendered/windows/self-mng-no-op.ps1 rename {examples/user_data => tests/user-data}/templates/al2023_custom.tpl (100%) rename {examples/user_data => tests/user-data}/templates/bottlerocket_custom.tpl (100%) rename {examples/user_data => tests/user-data}/templates/linux_custom.tpl (100%) rename {examples/user_data => tests/user-data}/templates/windows_custom.tpl (100%) create mode 100644 tests/user-data/variables.tf rename {examples/user_data => tests/user-data}/versions.tf (100%) diff --git a/README.md b/README.md index 124c57a830..901dff3675 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # AWS EKS Terraform module -Terraform module which creates AWS EKS (Kubernetes) resources +Terraform module which creates Amazon EKS (Kubernetes) resources [![SWUbanner](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner2-direct.svg)](https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md) @@ -23,13 +23,6 @@ Please note that we strive to provide a comprehensive suite of documentation for - [AWS EKS Documentation](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Kubernetes Documentation](https://kubernetes.io/docs/home/) -#### Reference Architecture - -The examples provided under `examples/` provide a comprehensive suite of configurations that demonstrate nearly all of the possible different configurations and settings that can be used with this module. However, these examples are not representative of clusters that you would normally find in use for production workloads. For reference architectures that utilize this module, please see the following: - -- [EKS Reference Architecture](https://github.com/clowdhaus/eks-reference-architecture) -- [EKS Blueprints](https://github.com/aws-ia/terraform-aws-eks-blueprints) - ## Usage ```hcl @@ -38,20 +31,15 @@ module "eks" { version = "~> 20.0" cluster_name = "my-cluster" - cluster_version = "1.29" + cluster_version = "1.30" cluster_endpoint_public_access = true cluster_addons = { - coredns = { - most_recent = true - } - kube-proxy = { - most_recent = true - } - vpc-cni = { - most_recent = true - } + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + vpc-cni = {} } vpc_id = "vpc-1234556abcdef" @@ -65,12 +53,13 @@ module "eks" { eks_managed_node_groups = { example = { - min_size = 1 - max_size = 10 - desired_size = 1 + # Starting on 1.30, AL2023 is the default AMI type for EKS managed node groups + ami_type = "AL2023_x86_64_STANDARD" + instance_types = ["m5.xlarge"] - instance_types = ["t3.large"] - capacity_type = "SPOT" + min_size = 2 + max_size = 10 + desired_size = 2 } } @@ -105,7 +94,7 @@ module "eks" { ### Cluster Access Entry -When enabling `authentication_mode = "API_AND_CONFIG_MAP"`, EKS will automatically create an access entry for the IAM role(s) used by managed nodegroup(s) and Fargate profile(s). There are no additional actions required by users. For self-managed nodegroups and the Karpenter sub-module, this project automatically adds the access entry on behalf of users so there are no additional actions required by users. +When enabling `authentication_mode = "API_AND_CONFIG_MAP"`, EKS will automatically create an access entry for the IAM role(s) used by managed node group(s) and Fargate profile(s). There are no additional actions required by users. For self-managed node groups and the Karpenter sub-module, this project automatically adds the access entry on behalf of users so there are no additional actions required by users. On clusters that were created prior to CAM support, there will be an existing access entry for the cluster creator. This was previously not visible when using `aws-auth` ConfigMap, but will become visible when access entry is enabled. @@ -115,10 +104,10 @@ Setting the `bootstrap_cluster_creator_admin_permissions` is a one time operatio ### Enabling EFA Support -When enabling EFA support via `enable_efa_support = true`, there are two locations this can be specified - one at the cluster level, and one at the nodegroup level. Enabling at the cluster level will add the EFA required ingress/egress rules to the shared security group created for the nodegroup(s). Enabling at the nodegroup level will do the following (per nodegroup where enabled): +When enabling EFA support via `enable_efa_support = true`, there are two locations this can be specified - one at the cluster level, and one at the node group level. Enabling at the cluster level will add the EFA required ingress/egress rules to the shared security group created for the node group(s). Enabling at the node group level will do the following (per node group where enabled): -1. All EFA interfaces supported by the instance will be exposed on the launch template used by the nodegroup -2. A placement group with `strategy = "clustered"` per EFA requirements is created and passed to the launch template used by the nodegroup +1. All EFA interfaces supported by the instance will be exposed on the launch template used by the node group +2. A placement group with `strategy = "clustered"` per EFA requirements is created and passed to the launch template used by the node group 3. Data sources will reverse lookup the availability zones that support the instance type selected based on the subnets provided, ensuring that only the associated subnets are passed to the launch template and therefore used by the placement group. This avoids the placement group being created in an availability zone that does not support the instance type selected. > [!TIP] @@ -126,11 +115,11 @@ When enabling EFA support via `enable_efa_support = true`, there are two locatio > > The EKS AL2 GPU AMI comes with the necessary EFA components pre-installed - you just need to expose the EFA devices on the nodes via their launch templates, ensure the required EFA security group rules are in place, and deploy the `aws-efa-k8s-device-plugin` in order to start utilizing EFA within your cluster. Your application container will need to have the necessary libraries and runtime in order to utilize communication over the EFA interfaces (NCCL, aws-ofi-nccl, hwloc, libfabric, aws-neuornx-collectives, CUDA, etc.). -If you disable the creation and use of the managed nodegroup custom launch template (`create_launch_template = false` and/or `use_custom_launch_template = false`), this will interfere with the EFA functionality provided. In addition, if you do not supply an `instance_type` for self-managed nodegroup(s), or `instance_types` for the managed nodegroup(s), this will also interfere with the functionality. In order to support the EFA functionality provided by `enable_efa_support = true`, you must utilize the custom launch template created/provided by this module, and supply an `instance_type`/`instance_types` for the respective nodegroup. +If you disable the creation and use of the managed node group custom launch template (`create_launch_template = false` and/or `use_custom_launch_template = false`), this will interfere with the EFA functionality provided. In addition, if you do not supply an `instance_type` for self-managed node group(s), or `instance_types` for the managed node group(s), this will also interfere with the functionality. In order to support the EFA functionality provided by `enable_efa_support = true`, you must utilize the custom launch template created/provided by this module, and supply an `instance_type`/`instance_types` for the respective node group. -The logic behind supporting EFA uses a data source to lookup the instance type to retrieve the number of interfaces that the instance supports in order to enumerate and expose those interfaces on the launch template created. For managed nodegroups where a list of instance types are supported, the first instance type in the list is used to calculate the number of EFA interfaces supported. Mixing instance types with varying number of interfaces is not recommended for EFA (or in some cases, mixing instance types is not supported - i.e. - p5.48xlarge and p4d.24xlarge). In addition to exposing the EFA interfaces and updating the security group rules, a placement group is created per the EFA requirements and only the availability zones that support the instance type selected are used in the subnets provided to the nodegroup. +The logic behind supporting EFA uses a data source to lookup the instance type to retrieve the number of interfaces that the instance supports in order to enumerate and expose those interfaces on the launch template created. For managed node groups where a list of instance types are supported, the first instance type in the list is used to calculate the number of EFA interfaces supported. Mixing instance types with varying number of interfaces is not recommended for EFA (or in some cases, mixing instance types is not supported - i.e. - p5.48xlarge and p4d.24xlarge). In addition to exposing the EFA interfaces and updating the security group rules, a placement group is created per the EFA requirements and only the availability zones that support the instance type selected are used in the subnets provided to the node group. -In order to enable EFA support, you will have to specify `enable_efa_support = true` on both the cluster and each nodegroup that you wish to enable EFA support for: +In order to enable EFA support, you will have to specify `enable_efa_support = true` on both the cluster and each node group that you wish to enable EFA support for: ```hcl module "eks" { @@ -140,14 +129,14 @@ module "eks" { # Truncated for brevity ... # Adds the EFA required security group rules to the shared - # security group created for the nodegroup(s) + # security group created for the node group(s) enable_efa_support = true eks_managed_node_groups = { example = { instance_types = ["p5.48xlarge"] - # Exposes all EFA interfaces on the launch template created by the nodegroup(s) + # Exposes all EFA interfaces on the launch template created by the node group(s) # This would expose all 32 EFA interfaces for the p5.48xlarge instance type enable_efa_support = true @@ -169,12 +158,10 @@ module "eks" { ## Examples -- [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group): EKS Cluster using EKS managed node groups -- [Fargate Profile](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile): EKS cluster using [Fargate Profiles](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) +- [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-managed-node-group): EKS Cluster using EKS managed node groups - [Karpenter](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/karpenter): EKS Cluster with [Karpenter](https://karpenter.sh/) provisioned for intelligent data plane management - [Outposts](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/outposts): EKS local cluster provisioned on [AWS Outposts](https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts.html) -- [Self Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self_managed_node_group): EKS Cluster using self-managed node groups -- [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data): Various supported methods of providing necessary bootstrap scripts and configuration settings via user data +- [Self Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self-managed-node-group): EKS Cluster using self-managed node groups ## Contributing @@ -290,7 +277,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [create\_kms\_key](#input\_create\_kms\_key) | Controls if a KMS key for cluster encryption should be created | `bool` | `true` | no | | [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | | [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | -| [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed nodegroup(s), self-managed nodegroup(s), Fargate profile(s)) | `string` | `"30s"` | no | +| [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed node group(s), self-managed node group(s), Fargate profile(s)) | `string` | `"30s"` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | | [enable\_cluster\_creator\_admin\_permissions](#input\_enable\_cluster\_creator\_admin\_permissions) | Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry | `bool` | `false` | no | diff --git a/docs/UPGRADE-20.0.md b/docs/UPGRADE-20.0.md index 1f16712fda..6f53e326c2 100644 --- a/docs/UPGRADE-20.0.md +++ b/docs/UPGRADE-20.0.md @@ -19,8 +19,8 @@ To give users advanced notice and provide some future direction for this module, 1. The `aws-auth` sub-module will be removed entirely from the project. Since this sub-module is captured in the v20.x releases, users can continue using it even after the module moves forward with the next major version. The long term strategy and direction is cluster access entry and to rely only on the AWS Terraform provider. 2. The default value for `authentication_mode` will change to `API`. Aligning with point 1 above, this is a one way change, but users are free to specify the value of their choosing in place of this default (when the change is made). This module will proceed with an EKS API first strategy. -3. The launch template and autoscaling group usage contained within the EKS managed nodegroup and self-managed nodegroup sub-modules *might be replaced with the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module. At minimum, it makes sense to replace most of functionality in the self-managed nodegroup module with this external module, but its not yet clear if there is any benefit of using it in the EKS managed nodegroup sub-module. The interface that users interact with will stay the same, the changes will be internal to the implementation and we will do everything we can to keep the disruption to a minimum. -4. The `platform` variable will be replaced and instead `ami_type` will become the standard across both self-managed nodegroup(s) and EKS managed nodegroup(s). As EKS expands its portfolio of supported operating systems, the `ami_type` is better suited to associate the correct user data format to the respective OS. The `platform` variable is a legacy artifact of self-managed nodegroups but not as descriptive as the `ami_type`, and therefore it will be removed in favor of `ami_type`. +3. The launch template and autoscaling group usage contained within the EKS managed node group and self-managed node group sub-modules *might be replaced with the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module. At minimum, it makes sense to replace most of functionality in the self-managed node group module with this external module, but its not yet clear if there is any benefit of using it in the EKS managed node group sub-module. The interface that users interact with will stay the same, the changes will be internal to the implementation and we will do everything we can to keep the disruption to a minimum. +4. The `platform` variable will be replaced and instead `ami_type` will become the standard across both self-managed node group(s) and EKS managed node group(s). As EKS expands its portfolio of supported operating systems, the `ami_type` is better suited to associate the correct user data format to the respective OS. The `platform` variable is a legacy artifact of self-managed node groups but not as descriptive as the `ami_type`, and therefore it will be removed in favor of `ami_type`. ## Additional changes @@ -29,8 +29,8 @@ To give users advanced notice and provide some future direction for this module, - A module tag has been added to the cluster control plane - Support for cluster access entries. The `bootstrap_cluster_creator_admin_permissions` setting on the control plane has been hardcoded to `false` since this operation is a one time operation only at cluster creation per the EKS API. Instead, users can enable/disable `enable_cluster_creator_admin_permissions` at any time to achieve the same functionality. This takes the identity that Terraform is using to make API calls and maps it into a cluster admin via an access entry. For users on existing clusters, you will need to remove the default cluster administrator that was created by EKS prior to the cluster access entry APIs - see the section [`Removing the default cluster administrator`](https://aws.amazon.com/blogs/containers/a-deep-dive-into-simplified-amazon-eks-access-management-controls/) for more details. - Support for specifying the CloudWatch log group class (standard or infrequent access) - - Native support for Windows based managed nodegroups similar to AL2 and Bottlerocket - - Self-managed nodegroups now support `instance_maintenance_policy` and have added `max_healthy_percentage`, `scale_in_protected_instances`, and `standby_instances` arguments to the `instance_refresh.preferences` block + - Native support for Windows based managed node groups similar to AL2 and Bottlerocket + - Self-managed node groups now support `instance_maintenance_policy` and have added `max_healthy_percentage`, `scale_in_protected_instances`, and `standby_instances` arguments to the `instance_refresh.preferences` block ### Modified @@ -109,7 +109,7 @@ To give users advanced notice and provide some future direction for this module, - `create_access_entry` - `access_entry_type` - - Self-managed nodegroup + - Self-managed node group - `instance_maintenance_policy` - `create_access_entry` - `iam_role_arn` @@ -135,7 +135,7 @@ To give users advanced notice and provide some future direction for this module, - Karpenter - `node_access_entry_arn` - - Self-managed nodegroup + - Self-managed node group - `access_entry_arn` ## Upgrade Migrations @@ -239,13 +239,13 @@ terraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]' # include if T Once the configmap has been removed from the statefile, you can add the new `aws-auth` sub-module and copy the relevant definitions from the EKS module over to the new `aws-auth` sub-module definition (see before after diff above). > [!CAUTION] -> You will need to add entries to the `aws-auth` sub-module for any IAM roles used by nodegroups and/or Fargate profiles - the module no longer handles this in the background on behalf of users. +> You will need to add entries to the `aws-auth` sub-module for any IAM roles used by node groups and/or Fargate profiles - the module no longer handles this in the background on behalf of users. > > When you apply the changes with the new sub-module, the configmap in the cluster will get updated with the contents provided in the sub-module definition, so please be sure all of the necessary entries are added before applying the changes. ### authentication_mode = "API_AND_CONFIG_MAP" -When using `authentication_mode = "API_AND_CONFIG_MAP"` and there are entries that will remain in the configmap (entries that cannot be replaced by cluster access entry), you will first need to update the `authentication_mode` on the cluster to `"API_AND_CONFIG_MAP"`. To help make this upgrade process easier, a copy of the changes defined in the [`v20.0.0`](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2858) PR have been captured [here](https://github.com/clowdhaus/terraform-aws-eks-v20-migrate) but with the `aws-auth` components still provided in the module. This means you get the equivalent of the `v20.0.0` module, but it still includes support for the `aws-auth` configmap. You can follow the provided README on that interim migration module for the order of execution and return here once the `authentication_mode` has been updated to `"API_AND_CONFIG_MAP"`. Note - EKS automatically adds access entries for the roles used by EKS managed nodegroups and Fargate profiles; users do not need to do anything additional for these roles. +When using `authentication_mode = "API_AND_CONFIG_MAP"` and there are entries that will remain in the configmap (entries that cannot be replaced by cluster access entry), you will first need to update the `authentication_mode` on the cluster to `"API_AND_CONFIG_MAP"`. To help make this upgrade process easier, a copy of the changes defined in the [`v20.0.0`](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2858) PR have been captured [here](https://github.com/clowdhaus/terraform-aws-eks-v20-migrate) but with the `aws-auth` components still provided in the module. This means you get the equivalent of the `v20.0.0` module, but it still includes support for the `aws-auth` configmap. You can follow the provided README on that interim migration module for the order of execution and return here once the `authentication_mode` has been updated to `"API_AND_CONFIG_MAP"`. Note - EKS automatically adds access entries for the roles used by EKS managed node groups and Fargate profiles; users do not need to do anything additional for these roles. Once the `authentication_mode` has been updated, next you will need to remove the configmap from the statefile to avoid any disruptions: @@ -261,10 +261,10 @@ terraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]' # include if T If you are using Terraform `v1.7+`, you can utilize the [`remove`](https://developer.hashicorp.com/terraform/language/resources/syntax#removing-resources) to facilitate both the removal of the configmap through code. You can create a fork/clone of the provided [migration module](https://github.com/clowdhaus/terraform-aws-eks-migrate-v19-to-v20) and add the `remove` blocks and apply those changes before proceeding. We do not want to force users onto the bleeding edge with this module, so we have not included `remove` support at this time. -Once the configmap has been removed from the statefile, you can add the new `aws-auth` sub-module and copy the relevant definitions from the EKS module over to the new `aws-auth` sub-module definition (see before after diff above). When you apply the changes with the new sub-module, the configmap in the cluster will get updated with the contents provided in the sub-module definition, so please be sure all of the necessary entries are added before applying the changes. In the before/example above - the configmap would remove any entries for roles used by nodegroups and/or Fargate Profiles, but maintain the custom entries for users and roles passed into the module definition. +Once the configmap has been removed from the statefile, you can add the new `aws-auth` sub-module and copy the relevant definitions from the EKS module over to the new `aws-auth` sub-module definition (see before after diff above). When you apply the changes with the new sub-module, the configmap in the cluster will get updated with the contents provided in the sub-module definition, so please be sure all of the necessary entries are added before applying the changes. In the before/example above - the configmap would remove any entries for roles used by node groups and/or Fargate Profiles, but maintain the custom entries for users and roles passed into the module definition. ### authentication_mode = "API" -In order to switch to `API` only using cluster access entry, you first need to update the `authentication_mode` on the cluster to `API_AND_CONFIG_MAP` without modifying the `aws-auth` configmap. To help make this upgrade process easier, a copy of the changes defined in the [`v20.0.0`](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2858) PR have been captured [here](https://github.com/clowdhaus/terraform-aws-eks-v20-migrate) but with the `aws-auth` components still provided in the module. This means you get the equivalent of the `v20.0.0` module, but it still includes support for the `aws-auth` configmap. You can follow the provided README on that interim migration module for the order of execution and return here once the `authentication_mode` has been updated to `"API_AND_CONFIG_MAP"`. Note - EKS automatically adds access entries for the roles used by EKS managed nodegroups and Fargate profiles; users do not need to do anything additional for these roles. +In order to switch to `API` only using cluster access entry, you first need to update the `authentication_mode` on the cluster to `API_AND_CONFIG_MAP` without modifying the `aws-auth` configmap. To help make this upgrade process easier, a copy of the changes defined in the [`v20.0.0`](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2858) PR have been captured [here](https://github.com/clowdhaus/terraform-aws-eks-v20-migrate) but with the `aws-auth` components still provided in the module. This means you get the equivalent of the `v20.0.0` module, but it still includes support for the `aws-auth` configmap. You can follow the provided README on that interim migration module for the order of execution and return here once the `authentication_mode` has been updated to `"API_AND_CONFIG_MAP"`. Note - EKS automatically adds access entries for the roles used by EKS managed node groups and Fargate profiles; users do not need to do anything additional for these roles. Once the `authentication_mode` has been updated, you can update the `authentication_mode` on the cluster to `API` and remove the `aws-auth` configmap components. diff --git a/examples/README.md b/examples/README.md index f417c0adc0..8d54b3db38 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,8 +1,5 @@ # Examples -Please note - the examples provided serve two primary means: - -1. Show users working examples of the various ways in which the module can be configured and features supported -2. A means of testing/validating module changes +The examples provided demonstrate different cluster configurations that users can create with the modules provided. Please do not mistake the examples provided as "best practices". It is up to users to consult the AWS service documentation for best practices, usage recommendations, etc. diff --git a/examples/eks-managed-node-group/README.md b/examples/eks-managed-node-group/README.md new file mode 100644 index 0000000000..a75636480a --- /dev/null +++ b/examples/eks-managed-node-group/README.md @@ -0,0 +1,23 @@ +# EKS Managed Node Group Examples + +Configuration in this directory creates Amazon EKS clusters with EKS Managed Node Groups demonstrating different configurations: + +- `eks-al2.tf` demonstrates an EKS cluster using EKS managed node group that utilizes the EKS Amazon Linux 2 optimized AMI +- `eks-al2023.tf` demonstrates an EKS cluster using EKS managed node group that utilizes the EKS Amazon Linux 2023 optimized AMI +- `eks-bottlerocket.tf` demonstrates an EKS cluster using EKS managed node group that utilizes the Bottlerocket EKS optimized AMI + +See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for additional details on Amazon EKS managed node groups. + +The different cluster configuration examples provided are separated per file and independent of the other cluster configurations. + +## Usage + +To provision the provided configurations you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply --auto-approve +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. diff --git a/examples/eks-managed-node-group/eks-al2.tf b/examples/eks-managed-node-group/eks-al2.tf new file mode 100644 index 0000000000..2dfb5b019f --- /dev/null +++ b/examples/eks-managed-node-group/eks-al2.tf @@ -0,0 +1,34 @@ +module "eks_al2" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + cluster_name = "${local.name}-al2" + cluster_version = "1.30" + + # EKS Addons + cluster_addons = { + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + vpc-cni = {} + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + example = { + # Starting on 1.30, AL2023 is the default AMI type for EKS managed node groups + ami_type = "AL2_x86_64" + instance_types = ["m6i.large"] + + min_size = 2 + max_size = 5 + # This value is ignored after the initial creation + # https://github.com/bryantbiggs/eks-desired-size-hack + desired_size = 2 + } + } + + tags = local.tags +} diff --git a/examples/eks-managed-node-group/eks-al2023.tf b/examples/eks-managed-node-group/eks-al2023.tf new file mode 100644 index 0000000000..1b112d23a2 --- /dev/null +++ b/examples/eks-managed-node-group/eks-al2023.tf @@ -0,0 +1,52 @@ +module "eks_al2023" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + cluster_name = "${local.name}-al2023" + cluster_version = "1.30" + + # EKS Addons + cluster_addons = { + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + vpc-cni = {} + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + example = { + # Starting on 1.30, AL2023 is the default AMI type for EKS managed node groups + instance_types = ["m6i.large"] + + min_size = 2 + max_size = 5 + # This value is ignored after the initial creation + # https://github.com/bryantbiggs/eks-desired-size-hack + desired_size = 2 + + # This is not required - demonstrates how to pass additional configuration to nodeadm + # Ref https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/api/ + cloudinit_pre_nodeadm = [ + { + content_type = "application/node.eks.aws" + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + } + ] + } + } + + tags = local.tags +} diff --git a/examples/eks-managed-node-group/eks-bottlerocket.tf b/examples/eks-managed-node-group/eks-bottlerocket.tf new file mode 100644 index 0000000000..44efa593c0 --- /dev/null +++ b/examples/eks-managed-node-group/eks-bottlerocket.tf @@ -0,0 +1,52 @@ +module "eks_bottlerocket" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + cluster_name = "${local.name}-bottlerocket" + cluster_version = "1.30" + + # EKS Addons + cluster_addons = { + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + vpc-cni = {} + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + example = { + ami_type = "BOTTLEROCKET_x86_64" + instance_types = ["m6i.large"] + + min_size = 2 + max_size = 5 + # This value is ignored after the initial creation + # https://github.com/bryantbiggs/eks-desired-size-hack + desired_size = 2 + + # This is not required - demonstrates how to pass additional configuration + # Ref https://bottlerocket.dev/en/os/1.19.x/api/settings/ + bootstrap_extra_args = <<-EOT + # The admin host container provides SSH access and runs with "superpowers". + # It is disabled by default, but can be disabled explicitly. + [settings.host-containers.admin] + enabled = false + + # The control host container provides out-of-band access via SSM. + # It is enabled by default, and can be disabled if you do not expect to use SSM. + # This could leave you with no way to access the API and change settings on an existing node! + [settings.host-containers.control] + enabled = true + + # extra args added + [settings.kernel] + lockdown = "integrity" + EOT + } + } + + tags = local.tags +} diff --git a/examples/eks-managed-node-group/main.tf b/examples/eks-managed-node-group/main.tf new file mode 100644 index 0000000000..4409a2e388 --- /dev/null +++ b/examples/eks-managed-node-group/main.tf @@ -0,0 +1,49 @@ +provider "aws" { + region = local.region +} + +data "aws_availability_zones" "available" {} + +locals { + name = "ex-eks-mng" + region = "eu-west-1" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# VPC +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/examples/eks_managed_node_group/variables.tf b/examples/eks-managed-node-group/outputs.tf similarity index 100% rename from examples/eks_managed_node_group/variables.tf rename to examples/eks-managed-node-group/outputs.tf diff --git a/examples/fargate_profile/variables.tf b/examples/eks-managed-node-group/variables.tf similarity index 100% rename from examples/fargate_profile/variables.tf rename to examples/eks-managed-node-group/variables.tf diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks-managed-node-group/versions.tf similarity index 100% rename from examples/eks_managed_node_group/versions.tf rename to examples/eks-managed-node-group/versions.tf diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index def4eb9f21..bc09df1fa5 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -4,12 +4,12 @@ Configuration in this directory creates an AWS EKS cluster with [Karpenter](http ## Usage -To run this example you need to execute: +To provision the provided configurations you need to execute: ```bash $ terraform init $ terraform plan -$ terraform apply +$ terraform apply --auto-approve ``` Once the cluster is up and running, you can check that Karpenter is functioning as intended with the following command: @@ -78,7 +78,7 @@ kubectl delete node -l karpenter.sh/provisioner-name=default 2. Remove the resources created by Terraform ```bash -terraform destroy +terraform destroy --auto-approve ``` Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index b3db54453e..9dd361792b 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -62,7 +62,7 @@ module "eks" { source = "../.." cluster_name = local.name - cluster_version = "1.29" + cluster_version = "1.30" # Gives Terraform identity admin access to cluster which will # allow deploying resources (Karpenter) into the cluster @@ -82,6 +82,7 @@ module "eks" { eks_managed_node_groups = { karpenter = { + ami_type = "AL2023_x86_64_STANDARD" instance_types = ["m5.large"] min_size = 2 @@ -146,7 +147,7 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "0.36.1" + version = "0.37.0" wait = false values = [ @@ -168,7 +169,7 @@ resource "kubectl_manifest" "karpenter_node_class" { metadata: name: default spec: - amiFamily: AL2 + amiFamily: AL2023 role: ${module.karpenter.node_iam_role_name} subnetSelectorTerms: - tags: diff --git a/examples/outposts/README.md b/examples/outposts/README.md index a454fd2c90..3779487fde 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -1,4 +1,4 @@ -# EKS on Outposts +# EKS on Outposts Example Configuration in this directory creates an AWS EKS local cluster on AWS Outposts @@ -16,7 +16,7 @@ To run this example you need to: $ cd prerequisites $ terraform init $ terraform plan -$ terraform apply +$ terraform apply --auto-approve ``` 2. If provisioning using the remote host deployed in step 1, connect to the remote host using SSM. Note, you will need to have the [SSM plugin for the AWS CLI installed](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html). You can use the output generated by step 1 to connect: @@ -31,13 +31,13 @@ $ aws ssm start-session --region --target $ cd $HOME/terraform-aws-eks/examples/outposts $ terraform init $ terraform plan -$ terraform apply +$ terraform apply --auto-approve ``` Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. ```bash -terraform destroy +terraform destroy --auto-approve ``` diff --git a/examples/outposts/main.tf b/examples/outposts/main.tf index 4b13f52465..c7835971f7 100644 --- a/examples/outposts/main.tf +++ b/examples/outposts/main.tf @@ -4,7 +4,7 @@ provider "aws" { locals { name = "ex-${basename(path.cwd)}" - cluster_version = "1.29" + cluster_version = "1.30" outpost_arn = element(tolist(data.aws_outposts_outposts.this.arns), 0) instance_type = element(tolist(data.aws_outposts_outpost_instance_types.this.instance_types), 0) diff --git a/examples/outposts/prerequisites/main.tf b/examples/outposts/prerequisites/main.tf index 66ab2a4e29..1a1dd18a4b 100644 --- a/examples/outposts/prerequisites/main.tf +++ b/examples/outposts/prerequisites/main.tf @@ -56,7 +56,7 @@ module "ssm_bastion_ec2" { rm terraform_${local.terraform_version}_linux_amd64.zip 2> /dev/null # Install kubectl - curl -LO https://dl.k8s.io/release/v1.29.0/bin/linux/amd64/kubectl + curl -LO https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl # Remove default awscli which is v1 - we want latest v2 diff --git a/examples/self-managed-node-group/README.md b/examples/self-managed-node-group/README.md new file mode 100644 index 0000000000..da81c84142 --- /dev/null +++ b/examples/self-managed-node-group/README.md @@ -0,0 +1,21 @@ +# Self-managed Node Group Examples + +Configuration in this directory creates Amazon EKS clusters with self-managed node groups demonstrating different configurations: + +- `eks-al2.tf` demonstrates an EKS cluster using self-managed node group that utilizes the EKS Amazon Linux 2 optimized AMI +- `eks-al2023.tf` demonstrates an EKS cluster using self-managed node group that utilizes the EKS Amazon Linux 2023 optimized AMI +- `eks-bottlerocket.tf` demonstrates an EKS cluster using self-managed node group that utilizes the Bottlerocket EKS optimized AMI + +The different cluster configuration examples provided are separated per file and independent of the other cluster configurations. + +## Usage + +To provision the provided configurations you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply --auto-approve +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. diff --git a/examples/self-managed-node-group/eks-al2.tf b/examples/self-managed-node-group/eks-al2.tf new file mode 100644 index 0000000000..be5c65ade5 --- /dev/null +++ b/examples/self-managed-node-group/eks-al2.tf @@ -0,0 +1,33 @@ +module "eks_al2" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + cluster_name = "${local.name}-al2" + cluster_version = "1.30" + + # EKS Addons + cluster_addons = { + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + vpc-cni = {} + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + self_managed_node_groups = { + example = { + ami_type = "AL2_x86_64" + instance_type = "m6i.large" + + min_size = 2 + max_size = 5 + # This value is ignored after the initial creation + # https://github.com/bryantbiggs/eks-desired-size-hack + desired_size = 2 + } + } + + tags = local.tags +} diff --git a/examples/self-managed-node-group/eks-al2023.tf b/examples/self-managed-node-group/eks-al2023.tf new file mode 100644 index 0000000000..701560527f --- /dev/null +++ b/examples/self-managed-node-group/eks-al2023.tf @@ -0,0 +1,52 @@ +module "eks_al2023" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + cluster_name = "${local.name}-al2023" + cluster_version = "1.30" + + # EKS Addons + cluster_addons = { + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + vpc-cni = {} + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + self_managed_node_groups = { + example = { + ami_type = "AL2023_x86_64_STANDARD" + instance_type = "m6i.large" + + min_size = 2 + max_size = 5 + # This value is ignored after the initial creation + # https://github.com/bryantbiggs/eks-desired-size-hack + desired_size = 2 + + # This is not required - demonstrates how to pass additional configuration to nodeadm + # Ref https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/api/ + cloudinit_pre_nodeadm = [ + { + content_type = "application/node.eks.aws" + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + } + ] + } + } + + tags = local.tags +} diff --git a/examples/self-managed-node-group/eks-bottlerocket.tf b/examples/self-managed-node-group/eks-bottlerocket.tf new file mode 100644 index 0000000000..2afb079472 --- /dev/null +++ b/examples/self-managed-node-group/eks-bottlerocket.tf @@ -0,0 +1,52 @@ +module "eks_bottlerocket" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + cluster_name = "${local.name}-bottlerocket" + cluster_version = "1.30" + + # EKS Addons + cluster_addons = { + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + vpc-cni = {} + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + self_managed_node_groups = { + example = { + ami_type = "BOTTLEROCKET_x86_64" + instance_type = "m6i.large" + + min_size = 2 + max_size = 5 + # This value is ignored after the initial creation + # https://github.com/bryantbiggs/eks-desired-size-hack + desired_size = 2 + + # This is not required - demonstrates how to pass additional configuration + # Ref https://bottlerocket.dev/en/os/1.19.x/api/settings/ + bootstrap_extra_args = <<-EOT + # The admin host container provides SSH access and runs with "superpowers". + # It is disabled by default, but can be disabled explicitly. + [settings.host-containers.admin] + enabled = false + + # The control host container provides out-of-band access via SSM. + # It is enabled by default, and can be disabled if you do not expect to use SSM. + # This could leave you with no way to access the API and change settings on an existing node! + [settings.host-containers.control] + enabled = true + + # extra args added + [settings.kernel] + lockdown = "integrity" + EOT + } + } + + tags = local.tags +} diff --git a/examples/self-managed-node-group/main.tf b/examples/self-managed-node-group/main.tf new file mode 100644 index 0000000000..ed6982aa58 --- /dev/null +++ b/examples/self-managed-node-group/main.tf @@ -0,0 +1,49 @@ +provider "aws" { + region = local.region +} + +data "aws_availability_zones" "available" {} + +locals { + name = "ex-self-mng" + region = "eu-west-1" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Example = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# VPC +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/examples/self_managed_node_group/variables.tf b/examples/self-managed-node-group/outputs.tf similarity index 100% rename from examples/self_managed_node_group/variables.tf rename to examples/self-managed-node-group/outputs.tf diff --git a/examples/user_data/variables.tf b/examples/self-managed-node-group/variables.tf similarity index 100% rename from examples/user_data/variables.tf rename to examples/self-managed-node-group/variables.tf diff --git a/examples/fargate_profile/versions.tf b/examples/self-managed-node-group/versions.tf similarity index 100% rename from examples/fargate_profile/versions.tf rename to examples/self-managed-node-group/versions.tf diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index b695ba6cca..79b8cbae5d 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -86,7 +86,7 @@ locals { } # https://github.com/aws/containers-roadmap/issues/596#issuecomment-675097667 -# Managed nodegroup data must in MIME multi-part archive format, +# Managed node group data must in MIME multi-part archive format, # as by default, EKS will merge the bootstrapping command required for nodes to join the # cluster with your user data. If you use a custom AMI in your launch template, # this merging will NOT happen and you are responsible for nodes joining the cluster. diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 9cc5680b84..494b187914 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -122,7 +122,7 @@ module "eks_managed_node_group" { | [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | -| [create\_placement\_group](#input\_create\_placement\_group) | Determines whether a placement group is created & used by the nodegroup | `bool` | `false` | no | +| [create\_placement\_group](#input\_create\_placement\_group) | Determines whether a placement group is created & used by the node group | `bool` | `false` | no | | [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | | [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | | [desired\_size](#input\_desired\_size) | Desired number of instances/nodes | `number` | `1` | no | diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 344b346ad1..d62629627f 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -298,7 +298,7 @@ variable "placement" { } variable "create_placement_group" { - description = "Determines whether a placement group is created & used by the nodegroup" + description = "Determines whether a placement group is created & used by the node group" type = bool default = false } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index ec819ed256..ee8cce3b12 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -43,7 +43,7 @@ In the following example, the Karpenter module will create: - An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller - SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. -In this scenario, Karpenter will re-use an existing Node IAM role from the EKS managed nodegroup which already has the necessary access entry permissions: +In this scenario, Karpenter will re-use an existing Node IAM role from the EKS managed node group which already has the necessary access entry permissions: ```hcl module "eks" { @@ -70,7 +70,7 @@ module "karpenter" { create_node_iam_role = false node_iam_role_arn = module.eks.eks_managed_node_groups["initial"].iam_role_arn - # Since the nodegroup role will already have an access entry + # Since the node group role will already have an access entry create_access_entry = false tags = { diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 5fb9cd3212..681a322b05 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -103,7 +103,7 @@ module "self_managed_node_group" { | [context](#input\_context) | Reserved | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | | [create](#input\_create) | Determines whether to create self managed node group or not | `bool` | `true` | no | -| [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the nodegroup | `bool` | `true` | no | +| [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the node group | `bool` | `true` | no | | [create\_autoscaling\_group](#input\_create\_autoscaling\_group) | Determines whether to create autoscaling group or not | `bool` | `true` | no | | [create\_iam\_instance\_profile](#input\_create\_iam\_instance\_profile) | Determines whether an IAM instance profile is created or to use an existing IAM instance profile | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create launch template or not | `bool` | `true` | no | diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index aea1269080..c4767479cd 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -650,7 +650,7 @@ variable "iam_role_tags" { ################################################################################ variable "create_access_entry" { - description = "Determines whether an access entry is created for the IAM role used by the nodegroup" + description = "Determines whether an access entry is created for the IAM role used by the node group" type = bool default = true } diff --git a/examples/eks_managed_node_group/README.md b/tests/eks-managed-node-group/README.md similarity index 88% rename from examples/eks_managed_node_group/README.md rename to tests/eks-managed-node-group/README.md index a6faed88e6..169b4c4f18 100644 --- a/examples/eks_managed_node_group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -1,25 +1,13 @@ -# EKS Managed Node Group Example - -Configuration in this directory creates an AWS EKS cluster with various EKS Managed Node Groups demonstrating the various methods of configuring/customizing: - -- A default, "out of the box" EKS managed node group as supplied by AWS EKS -- A default, "out of the box" Bottlerocket EKS managed node group as supplied by AWS EKS -- A Bottlerocket EKS managed node group that supplies additional bootstrap settings -- A Bottlerocket EKS managed node group that demonstrates many of the configuration/customizations offered by the `eks-managed-node-group` sub-module for the Bottlerocket OS -- An EKS managed node group created from a launch template created outside of the module -- An EKS managed node group that utilizes a custom AMI that is an EKS optimized AMI derivative -- An EKS managed node group that demonstrates nearly all of the configurations/customizations offered by the `eks-managed-node-group` sub-module - -See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for further details. +# EKS Managed Node Group ## Usage -To run this example you need to execute: +To provision the provided configurations you need to execute: ```bash $ terraform init $ terraform plan -$ terraform apply +$ terraform apply --auto-approve ``` Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. diff --git a/examples/eks_managed_node_group/main.tf b/tests/eks-managed-node-group/main.tf similarity index 99% rename from examples/eks_managed_node_group/main.tf rename to tests/eks-managed-node-group/main.tf index 2f1202f5c5..c5bb833d14 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -14,7 +14,7 @@ locals { azs = slice(data.aws_availability_zones.available.names, 0, 3) tags = { - Example = local.name + Test = local.name GithubRepo = "terraform-aws-eks" GithubOrg = "terraform-aws-modules" } diff --git a/examples/eks_managed_node_group/outputs.tf b/tests/eks-managed-node-group/outputs.tf similarity index 100% rename from examples/eks_managed_node_group/outputs.tf rename to tests/eks-managed-node-group/outputs.tf diff --git a/examples/user_data/rendered/al2/eks-mng-no-op.sh b/tests/eks-managed-node-group/variables.tf old mode 100755 new mode 100644 similarity index 100% rename from examples/user_data/rendered/al2/eks-mng-no-op.sh rename to tests/eks-managed-node-group/variables.tf diff --git a/examples/self_managed_node_group/versions.tf b/tests/eks-managed-node-group/versions.tf similarity index 100% rename from examples/self_managed_node_group/versions.tf rename to tests/eks-managed-node-group/versions.tf diff --git a/examples/fargate_profile/README.md b/tests/fargate-profile/README.md similarity index 97% rename from examples/fargate_profile/README.md rename to tests/fargate-profile/README.md index 604fd3a8ca..bdb1a909a5 100644 --- a/examples/fargate_profile/README.md +++ b/tests/fargate-profile/README.md @@ -1,15 +1,13 @@ -# AWS EKS Cluster with Fargate profiles - -Configuration in this directory creates an AWS EKS cluster utilizing Fargate profiles. +# Fargate Profile ## Usage -To run this example you need to execute: +To provision the provided configurations you need to execute: ```bash $ terraform init $ terraform plan -$ terraform apply +$ terraform apply --auto-approve ``` Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. diff --git a/examples/fargate_profile/main.tf b/tests/fargate-profile/main.tf similarity index 96% rename from examples/fargate_profile/main.tf rename to tests/fargate-profile/main.tf index 16fe82c1bc..6106022373 100644 --- a/examples/fargate_profile/main.tf +++ b/tests/fargate-profile/main.tf @@ -5,15 +5,15 @@ provider "aws" { data "aws_availability_zones" "available" {} locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.29" + name = "ex-${basename(path.cwd)}" + cluster_version = "1.30" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) tags = { - Example = local.name + Test = local.name GithubRepo = "terraform-aws-eks" GithubOrg = "terraform-aws-modules" } diff --git a/examples/fargate_profile/outputs.tf b/tests/fargate-profile/outputs.tf similarity index 100% rename from examples/fargate_profile/outputs.tf rename to tests/fargate-profile/outputs.tf diff --git a/examples/user_data/rendered/al2/self-mng-no-op.sh b/tests/fargate-profile/variables.tf old mode 100755 new mode 100644 similarity index 100% rename from examples/user_data/rendered/al2/self-mng-no-op.sh rename to tests/fargate-profile/variables.tf diff --git a/tests/fargate-profile/versions.tf b/tests/fargate-profile/versions.tf new file mode 100644 index 0000000000..6f83215f50 --- /dev/null +++ b/tests/fargate-profile/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.2" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.40" + } + } +} diff --git a/examples/self_managed_node_group/README.md b/tests/self-managed-node-group/README.md similarity index 90% rename from examples/self_managed_node_group/README.md rename to tests/self-managed-node-group/README.md index f6f063f693..feebef9c92 100644 --- a/examples/self_managed_node_group/README.md +++ b/tests/self-managed-node-group/README.md @@ -1,21 +1,13 @@ -# Self Managed Node Groups Example - -Configuration in this directory creates an AWS EKS cluster with various Self Managed Node Groups (AutoScaling Groups) demonstrating the various methods of configuring/customizing: - -- A default, "out of the box" self managed node group as supplied by the `self-managed-node-group` sub-module -- A Bottlerocket self managed node group that demonstrates many of the configuration/customizations offered by the `self-manged-node-group` sub-module for the Bottlerocket OS -- A self managed node group that demonstrates nearly all of the configurations/customizations offered by the `self-managed-node-group` sub-module - -See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for further details. +# Self-managed Node Group ## Usage -To run this example you need to execute: +To provision the provided configurations you need to execute: ```bash $ terraform init $ terraform plan -$ terraform apply +$ terraform apply --auto-approve ``` Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. diff --git a/examples/self_managed_node_group/main.tf b/tests/self-managed-node-group/main.tf similarity index 99% rename from examples/self_managed_node_group/main.tf rename to tests/self-managed-node-group/main.tf index 1d0e80b2fd..7fc6171e95 100644 --- a/examples/self_managed_node_group/main.tf +++ b/tests/self-managed-node-group/main.tf @@ -14,7 +14,7 @@ locals { azs = slice(data.aws_availability_zones.available.names, 0, 3) tags = { - Example = local.name + Test = local.name GithubRepo = "terraform-aws-eks" GithubOrg = "terraform-aws-modules" } diff --git a/examples/self_managed_node_group/outputs.tf b/tests/self-managed-node-group/outputs.tf similarity index 100% rename from examples/self_managed_node_group/outputs.tf rename to tests/self-managed-node-group/outputs.tf diff --git a/examples/user_data/rendered/al2023/eks-mng-no-op.txt b/tests/self-managed-node-group/variables.tf old mode 100755 new mode 100644 similarity index 100% rename from examples/user_data/rendered/al2023/eks-mng-no-op.txt rename to tests/self-managed-node-group/variables.tf diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf new file mode 100644 index 0000000000..6f83215f50 --- /dev/null +++ b/tests/self-managed-node-group/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.2" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.40" + } + } +} diff --git a/examples/user_data/README.md b/tests/user-data/README.md similarity index 99% rename from examples/user_data/README.md rename to tests/user-data/README.md index de9b419bb4..3278c7f0f9 100644 --- a/examples/user_data/README.md +++ b/tests/user-data/README.md @@ -4,12 +4,12 @@ Configuration in this directory render various user data outputs used for testin ## Usage -To run this example you need to execute: +To provision the provided configurations you need to execute: ```bash $ terraform init $ terraform plan -$ terraform apply +$ terraform apply --auto-approve ``` diff --git a/examples/user_data/main.tf b/tests/user-data/main.tf similarity index 100% rename from examples/user_data/main.tf rename to tests/user-data/main.tf diff --git a/examples/user_data/outputs.tf b/tests/user-data/outputs.tf similarity index 100% rename from examples/user_data/outputs.tf rename to tests/user-data/outputs.tf diff --git a/examples/user_data/rendered/al2/eks-mng-additional.txt b/tests/user-data/rendered/al2/eks-mng-additional.txt similarity index 100% rename from examples/user_data/rendered/al2/eks-mng-additional.txt rename to tests/user-data/rendered/al2/eks-mng-additional.txt diff --git a/examples/user_data/rendered/al2/eks-mng-custom-ami-ipv6.sh b/tests/user-data/rendered/al2/eks-mng-custom-ami-ipv6.sh similarity index 100% rename from examples/user_data/rendered/al2/eks-mng-custom-ami-ipv6.sh rename to tests/user-data/rendered/al2/eks-mng-custom-ami-ipv6.sh diff --git a/examples/user_data/rendered/al2/eks-mng-custom-ami.sh b/tests/user-data/rendered/al2/eks-mng-custom-ami.sh similarity index 100% rename from examples/user_data/rendered/al2/eks-mng-custom-ami.sh rename to tests/user-data/rendered/al2/eks-mng-custom-ami.sh diff --git a/examples/user_data/rendered/al2/eks-mng-custom-template.sh b/tests/user-data/rendered/al2/eks-mng-custom-template.sh similarity index 100% rename from examples/user_data/rendered/al2/eks-mng-custom-template.sh rename to tests/user-data/rendered/al2/eks-mng-custom-template.sh diff --git a/examples/user_data/rendered/al2023/self-mng-no-op.txt b/tests/user-data/rendered/al2/eks-mng-no-op.sh similarity index 100% rename from examples/user_data/rendered/al2023/self-mng-no-op.txt rename to tests/user-data/rendered/al2/eks-mng-no-op.sh diff --git a/examples/user_data/rendered/al2/self-mng-bootstrap-ipv6.sh b/tests/user-data/rendered/al2/self-mng-bootstrap-ipv6.sh similarity index 100% rename from examples/user_data/rendered/al2/self-mng-bootstrap-ipv6.sh rename to tests/user-data/rendered/al2/self-mng-bootstrap-ipv6.sh diff --git a/examples/user_data/rendered/al2/self-mng-bootstrap.sh b/tests/user-data/rendered/al2/self-mng-bootstrap.sh similarity index 100% rename from examples/user_data/rendered/al2/self-mng-bootstrap.sh rename to tests/user-data/rendered/al2/self-mng-bootstrap.sh diff --git a/examples/user_data/rendered/al2/self-mng-custom-template.sh b/tests/user-data/rendered/al2/self-mng-custom-template.sh similarity index 100% rename from examples/user_data/rendered/al2/self-mng-custom-template.sh rename to tests/user-data/rendered/al2/self-mng-custom-template.sh diff --git a/examples/user_data/rendered/bottlerocket/eks-mng-no-op.toml b/tests/user-data/rendered/al2/self-mng-no-op.sh similarity index 100% rename from examples/user_data/rendered/bottlerocket/eks-mng-no-op.toml rename to tests/user-data/rendered/al2/self-mng-no-op.sh diff --git a/examples/user_data/rendered/al2023/eks-mng-additional.txt b/tests/user-data/rendered/al2023/eks-mng-additional.txt similarity index 100% rename from examples/user_data/rendered/al2023/eks-mng-additional.txt rename to tests/user-data/rendered/al2023/eks-mng-additional.txt diff --git a/examples/user_data/rendered/al2023/eks-mng-custom-ami.txt b/tests/user-data/rendered/al2023/eks-mng-custom-ami.txt similarity index 100% rename from examples/user_data/rendered/al2023/eks-mng-custom-ami.txt rename to tests/user-data/rendered/al2023/eks-mng-custom-ami.txt diff --git a/examples/user_data/rendered/al2023/eks-mng-custom-template.txt b/tests/user-data/rendered/al2023/eks-mng-custom-template.txt similarity index 100% rename from examples/user_data/rendered/al2023/eks-mng-custom-template.txt rename to tests/user-data/rendered/al2023/eks-mng-custom-template.txt diff --git a/examples/user_data/rendered/bottlerocket/self-mng-no-op.toml b/tests/user-data/rendered/al2023/eks-mng-no-op.txt similarity index 100% rename from examples/user_data/rendered/bottlerocket/self-mng-no-op.toml rename to tests/user-data/rendered/al2023/eks-mng-no-op.txt diff --git a/examples/user_data/rendered/al2023/self-mng-bootstrap.txt b/tests/user-data/rendered/al2023/self-mng-bootstrap.txt similarity index 100% rename from examples/user_data/rendered/al2023/self-mng-bootstrap.txt rename to tests/user-data/rendered/al2023/self-mng-bootstrap.txt diff --git a/examples/user_data/rendered/al2023/self-mng-custom-template.txt b/tests/user-data/rendered/al2023/self-mng-custom-template.txt similarity index 100% rename from examples/user_data/rendered/al2023/self-mng-custom-template.txt rename to tests/user-data/rendered/al2023/self-mng-custom-template.txt diff --git a/examples/user_data/rendered/windows/eks-mng-no-op.ps1 b/tests/user-data/rendered/al2023/self-mng-no-op.txt similarity index 100% rename from examples/user_data/rendered/windows/eks-mng-no-op.ps1 rename to tests/user-data/rendered/al2023/self-mng-no-op.txt diff --git a/examples/user_data/rendered/bottlerocket/eks-mng-additional.toml b/tests/user-data/rendered/bottlerocket/eks-mng-additional.toml similarity index 100% rename from examples/user_data/rendered/bottlerocket/eks-mng-additional.toml rename to tests/user-data/rendered/bottlerocket/eks-mng-additional.toml diff --git a/examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml b/tests/user-data/rendered/bottlerocket/eks-mng-custom-ami.toml similarity index 100% rename from examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml rename to tests/user-data/rendered/bottlerocket/eks-mng-custom-ami.toml diff --git a/examples/user_data/rendered/bottlerocket/eks-mng-custom-template.toml b/tests/user-data/rendered/bottlerocket/eks-mng-custom-template.toml similarity index 100% rename from examples/user_data/rendered/bottlerocket/eks-mng-custom-template.toml rename to tests/user-data/rendered/bottlerocket/eks-mng-custom-template.toml diff --git a/examples/user_data/rendered/windows/self-mng-no-op.ps1 b/tests/user-data/rendered/bottlerocket/eks-mng-no-op.toml similarity index 100% rename from examples/user_data/rendered/windows/self-mng-no-op.ps1 rename to tests/user-data/rendered/bottlerocket/eks-mng-no-op.toml diff --git a/examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml b/tests/user-data/rendered/bottlerocket/self-mng-bootstrap.toml similarity index 100% rename from examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml rename to tests/user-data/rendered/bottlerocket/self-mng-bootstrap.toml diff --git a/examples/user_data/rendered/bottlerocket/self-mng-custom-template.toml b/tests/user-data/rendered/bottlerocket/self-mng-custom-template.toml similarity index 100% rename from examples/user_data/rendered/bottlerocket/self-mng-custom-template.toml rename to tests/user-data/rendered/bottlerocket/self-mng-custom-template.toml diff --git a/tests/user-data/rendered/bottlerocket/self-mng-no-op.toml b/tests/user-data/rendered/bottlerocket/self-mng-no-op.toml new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/rendered/windows/eks-mng-additional.ps1 b/tests/user-data/rendered/windows/eks-mng-additional.ps1 similarity index 100% rename from examples/user_data/rendered/windows/eks-mng-additional.ps1 rename to tests/user-data/rendered/windows/eks-mng-additional.ps1 diff --git a/examples/user_data/rendered/windows/eks-mng-custom-ami.ps1 b/tests/user-data/rendered/windows/eks-mng-custom-ami.ps1 similarity index 100% rename from examples/user_data/rendered/windows/eks-mng-custom-ami.ps1 rename to tests/user-data/rendered/windows/eks-mng-custom-ami.ps1 diff --git a/examples/user_data/rendered/windows/eks-mng-custom-template.ps1 b/tests/user-data/rendered/windows/eks-mng-custom-template.ps1 similarity index 100% rename from examples/user_data/rendered/windows/eks-mng-custom-template.ps1 rename to tests/user-data/rendered/windows/eks-mng-custom-template.ps1 diff --git a/tests/user-data/rendered/windows/eks-mng-no-op.ps1 b/tests/user-data/rendered/windows/eks-mng-no-op.ps1 new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/rendered/windows/self-mng-bootstrap.ps1 b/tests/user-data/rendered/windows/self-mng-bootstrap.ps1 similarity index 100% rename from examples/user_data/rendered/windows/self-mng-bootstrap.ps1 rename to tests/user-data/rendered/windows/self-mng-bootstrap.ps1 diff --git a/examples/user_data/rendered/windows/self-mng-custom-template.ps1 b/tests/user-data/rendered/windows/self-mng-custom-template.ps1 similarity index 100% rename from examples/user_data/rendered/windows/self-mng-custom-template.ps1 rename to tests/user-data/rendered/windows/self-mng-custom-template.ps1 diff --git a/tests/user-data/rendered/windows/self-mng-no-op.ps1 b/tests/user-data/rendered/windows/self-mng-no-op.ps1 new file mode 100755 index 0000000000..e69de29bb2 diff --git a/examples/user_data/templates/al2023_custom.tpl b/tests/user-data/templates/al2023_custom.tpl similarity index 100% rename from examples/user_data/templates/al2023_custom.tpl rename to tests/user-data/templates/al2023_custom.tpl diff --git a/examples/user_data/templates/bottlerocket_custom.tpl b/tests/user-data/templates/bottlerocket_custom.tpl similarity index 100% rename from examples/user_data/templates/bottlerocket_custom.tpl rename to tests/user-data/templates/bottlerocket_custom.tpl diff --git a/examples/user_data/templates/linux_custom.tpl b/tests/user-data/templates/linux_custom.tpl similarity index 100% rename from examples/user_data/templates/linux_custom.tpl rename to tests/user-data/templates/linux_custom.tpl diff --git a/examples/user_data/templates/windows_custom.tpl b/tests/user-data/templates/windows_custom.tpl similarity index 100% rename from examples/user_data/templates/windows_custom.tpl rename to tests/user-data/templates/windows_custom.tpl diff --git a/tests/user-data/variables.tf b/tests/user-data/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/user_data/versions.tf b/tests/user-data/versions.tf similarity index 100% rename from examples/user_data/versions.tf rename to tests/user-data/versions.tf diff --git a/variables.tf b/variables.tf index 639110a9fc..2d75a4b130 100644 --- a/variables.tf +++ b/variables.tf @@ -499,7 +499,7 @@ variable "cluster_encryption_policy_tags" { } variable "dataplane_wait_duration" { - description = "Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed nodegroup(s), self-managed nodegroup(s), Fargate profile(s))" + description = "Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed node group(s), self-managed node group(s), Fargate profile(s))" type = string default = "30s" } From 7cd3be3fbbb695105a447b37c4653a00b0b51b94 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 14 Jun 2024 09:46:46 -0400 Subject: [PATCH 791/962] docs: Add `mkdocs` support to render static documentation site on GitHub (#3070) * docs: Add `mkdocs` support to render static documentation site on GitHub * fix: Correct mis-spelled file name --- .github/workflows/publish-docs.yml | 41 ++++++++++++++++++ docs/.pages | 3 ++ docs/assets/logo.png | Bin 0 -> 12523 bytes docs/assets/terraform-aws.png | Bin 0 -> 37726 bytes docs/index.md | 3 ++ docs/local.md | 20 +++++++++ mkdocs.yml | 65 +++++++++++++++++++++++++++++ 7 files changed, 132 insertions(+) create mode 100644 .github/workflows/publish-docs.yml create mode 100644 docs/.pages create mode 100644 docs/assets/logo.png create mode 100644 docs/assets/terraform-aws.png create mode 100644 docs/index.md create mode 100644 docs/local.md create mode 100644 mkdocs.yml diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml new file mode 100644 index 0000000000..918601d915 --- /dev/null +++ b/.github/workflows/publish-docs.yml @@ -0,0 +1,41 @@ +name: Publish docs via GitHub Pages +on: + workflow_dispatch: + push: + branches: + - main + - master + +permissions: + contents: read + +jobs: + build: + name: Deploy docs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install mkdocs-material==9.5.26 \ + mkdocs-include-markdown-plugin==6.2.0 \ + mkdocs-awesome-pages-plugin==2.9.2 + + - name: git config + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + - run: mkdocs gh-deploy --force diff --git a/docs/.pages b/docs/.pages new file mode 100644 index 0000000000..ee23da4f7f --- /dev/null +++ b/docs/.pages @@ -0,0 +1,3 @@ +nav: + - Overview: index.md + - Local Develpment: local.md diff --git a/docs/assets/logo.png b/docs/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..cdfb9fc545bd8d22935377999d41c89c2365eaa4 GIT binary patch literal 12523 zcmbVzRa9I}u=U{X4grQhg1bX-4MBsuyA#|82`&l29fG^Ny9al7cenq1>;5nI^*+qZ znzPPyRdrYGy}M6Mn4-KC8Zt352n0g=DlPsU_#FK2g$NJ4muZw|fIuwOU&TdK+|y3d z5j2$Np9j`=%tX794F^^3uyuGUKe|}sU7FLYlxfwu4o+xkA02d9(OaFCzwnW2Sl89z z6={`e^jwbW9Hm3O9fA$xi!f4oh{E$jz7L7GdW|*+t6c+MU^F{%JUD zzyB_Okd*+#YD8gX#@7~cSwC(KyO@v!9uOt{7S^I+R4Y#*)pZrr1z%Du;|F;d|Aw|S zlb*_78W|x__f1Y^bfaD-4Dv7D6Owp{8(`uG zV8TwSr#qCoO9xK0lYvofTy9ClOTba}_NY{rD8eQXGe3kEMI3?hIp2Hi2_s1J*_9iW z67a`z5(hS?hut*hVdttZz6GNo{Q0*nYB$HhS7-`@Klt>y1AJ-b z7LWt;AvRb)hCdY3Kf6ZYC*aaHnvet?WZbAg4lceeQa(9Kc&kqC$#LQgaluc%9vnhX z8pct?l4-OzUR`Xeup;kZF@hnrqKu4c(eEzWfd;P5i+8~AXu$A;2uOd8_sT;n_+s!a zDPo1mqWy-+Ci502O$Vmoej5Ob5DSAHYRCd2jHif=1TSO4@`52?On71FLs(#8>ra*$ zuRbrMD}x!!7+C0zI#sJr4;U;UO8bPrtEHUj#lXCny!~5oNRw$1CL}Yf(%AnOP95Y5w<*^x$@jTc-J1^% zHmX^8z-ZD?Ku_>detQakpz8v%hpg?63NUSRICY)8_YqxKKspgS0kF(|UF`h)U@`u) z3+u$ogyg0auzdYZzP+ZjQt<^D#4=Q0KrbR7Js+^Us>C1-tX%{3Xg>jN=z)EQkKOE2 zU19|uemFr&Di7!utPtNszf~d`ELGX=`KWr#_{FOfViVXL9srF+&sb)&6iv!|4&$#` zQ6>WI(cEZ?_&3uW@vnVz5ha%%?ZE$K8En9+m>pWV6K1 z>?_~zLZj39<{s2{2uIMSw;Blsxugfy+T21u@Tt^RS1Nv~DnJ%Pz(yRJLA<1;^%7TR zgZ-Vk$58Fd4h|}!8c6^Gc3W$$Wd<4ODG>Zbr;4=`wXAa51oAtshf-r`yTF0#`GI{n zW44Z-{*&6eub1^&k@D_22((12k3kiC7OebJxcQ41SoeT@)ZI%p=apIEIbdkW?dQqD zuWNQtmRp%K2zjc7y7eUWD*c;Yo1fXb-6Wx5UHH^2!9vUZ*+O9#y|W;W-2Ef!`Quc4T2*Sx&= zTff6RV_JUKoitcbLxFX^$K)w28&804hZAWTW#VF*E36s}MM9Am8y_YYgJ%MKtOX@CQ+dv@TUmzv_9 z`IWuo^wJELC!(b!uBk(rdKh4Hj2&Q~m*$t#^uoh-(c%1COqK&#d`4x!eEEHj%-(gg zRetWU$F{hc;0)LrTto|MVv;SKllCh%6pb!rx|RZdDN+fn)R9 z`g$M)zeW7iT|N-ujHDxhl7Scw1i=uFo|*v@z(j}L!YKI$`RD@(^w(rL2uw~D z>)O~njR#o4A)sEk`{9^t3+FFcZ7Uj!uynUiff{XKjuCCZJUINPE!9PNE(BPj9e>y> zM8J#-6vZgL$ub0tuwz8=Klra;vKNM4)EALDujc} zG{%Am^zh48nS001k581vhcXE&X zpib~|`I&zRg1NDi(+n~I=*Y1Vw?Tm8d4S{UtuMv1U;^%?@{F|l?)@6qGCcwCpiVE8 zG&tQDDA(9Kzf@qhs({&2lVzYr1|bhtMRHW{k6x1AJf9nzMSWO2=34A`{si0)HjOz5 z0jNgE)O484o%E`fr{o8agwQGG+g_sfS0i_)l$3kpw=LrJ6bdj@vEvY^&VXBT#Ot}w zDz3P!SDfQq8x;bXP2fKw!#=nSy}fp1>a`W85&A{2oqMHVO)M0L=+7#tLVk z{#PIOYQ5@swreWv1+O82lnaX_aUb>=RpyFnmdc0ltB_S1bD_@tZj`C0g zL!22WUYpA{FHR1sVF1={y%lnBDcEM;*M4twBAplzN~*#H5f8FoS1AVnyyCK37i6;; zOMwXS@Kk!KB(SF-c<{BxR!ebE06ew~@YoZ;V+nec01oTmb+7zRWfW@WEV8o%;8zgj zFvQs)_Lqz-?KSRmlfCz)M8&ZQi5Pw%f`_MphZ`XYy;RmA#3$bUQ|jfjwnV_WNc(RR{(kAN1K_fAbPS zA0C4Yu2F5{Jlq|8Gx_K3bDQLD1ZV;G>K9B9|Bn3U9)rG4Q}h3<$R(01Jw8wzF#hJZ zcKxlqKfEd9;&Syww%jqPFn|9i`;tj@6Y2oDc5-(vNFDG5xR=wfP?d%ZO3duPee|B| z1AbHR-;6q(>2%x_IHLUkAqDBY#JtQtTmvHN_z?Q5!83;Wc!T_E@{+Xy05fGwE82|| z9DnYg^5w*Sj;0?c9vu#;Ix97qA%%-HBXj`)vVUI@=MQ@@0T3?dEd+3KJs*YY83?cm z2+#=RaMsHy4lKdW{Tj;aN1DNL6>9oS-a>>!K@C8B>MJZ#4*w^|T1sU#Q~RwQU|Sdj zyX;AGZRw|Y=3$dhq6nN+M4m{&ap=#-vC_yNikh1M=d-)O5b6Fu5see#_Jup-qg zd^Pb)<+6noF>kyl@Vd|IJlK~A*Cb?*Jp&Z3RrYxqKF%1BgmNH6&!35Zo)FmQ&OKX0>dq!TA|h5nvDQOvINvM)0*!M8t9*Ny;%p58TGt4`+lONDXL2r^J+C0CN;QywEI?Q32uM4$n@95Y;R=<@{~^R| zU(_8vV2BR^droYG^sm{Rh>6YZOOuay;R3NM<^qAdzK$3=C5NV93KR_-R2uMNV0N+md zw2fKH$TGM=gZ})s#n=*w6>f$C2uw=wrB{&ZOF$hm=nS}?g9k{yfEb}%uK$vEd`54L z$*xu$2-*S&?5B;a(*qb#3UDAeljW&t;8MOUFb6(>e;pTyYS7^dgD^;cNw&8q3L_LM zpPhTmd|5zLakdHyut>eI06NYCInHVC7*JNxK<+~VDjcjYD)5Ij5(Q{+zkIDXlN@IR z4aQ#<5KBm4ND>(EkGo^(h>#3t#0L%JG*my3mR&e&Rc-73xln~p_H!J2aGlE?x4mhH z*P&Iu&)F)pb+XD&tW0GZ+>Pr~Dv`JLx=olKdG}jaAqzu8iu`Fy8RKEKB8Spl3|vrT zco9qF(?Yr8b2SYvh&U=gOOb}uR|z0i6eWQ(63p(_xi|K_?-D(Zt#XvoxrWGe@m3e# z)M@_Q+@jPUq1z;HSjp~a80dAPv=|q@<2?OcVM5%aAaMb zmPU_|>(vQCVui_gx1)}7LLDA~QO*D}GSba^n9QjL6)6RFMA_XL|DP0(=W*$+v(SoU z$=t4zidGKl&WY#!=y9cz*+0Lhp6xnL7VodgpWmZZ=p4s>{2tl$-(LEL6B8~{fF%kA zpN@`8hi6ovAjp0jTvg%G*tEl*jGB8skISMBnc?JE`1|**=3wXWT6sKApICC`bG*zX ze*kn!`1F-uUH#$wvyc+)0V>~UI)|o6rjn8O~=nMIq+($;*l=bRQ`eN~(QQvy*<_yZOGs z2O(cM<&@&Apy(@d=I3$^wjYwC+@~>#(s?JDZm&%?Bhl-EuH6lrRC(o>;R?aGL90b5Fdr!M5_Syz$`|IrY%iH z;gO5rs|;13h&@wC4lP0t3k_oj2A=Bdc>kKvq_tjq0vVV7~AoMe61^X$jjS)7c0Ws+r7evfv9G4)J8l;YmG3`(;r%&c#(&Z68iyhPbW09RcYB*O4GY%W}d^J;hFkgRp zpx?}FcumV8BVMaWaR_$Cl)&8AZFl`qt*Ft%-M<$eKK!rsMf9Vjg>tl{yD%g$>Lx(53+gR>l*cYZ;OMvyJSq4%}s_0Z#(7C>hcHXtuirTH>KDQ zoiA=2%C;7lbt57Pb-1$hrSaA?+=4_3b~(9Z@b?JtwO?=V9VbfpJ$&ILKD7rKgdz*S zVB$pc8rL z6>vVO`P1>1oc#kM%zB$xY4hK}beqA+^#oGBEODHLxjCiDKO!f*a?+DR{5i>tFFw96 zL(2O)BlL7)-(=3OYbW&3_I7Gh^Mq+|k>zVWZSZk)bkYo#{#=fD@$%22lj6EgjgQyo z8ct%oytW#2FZ@zeWxRR(@xGbZvEFPQ)mZg)qAkl`r#wiZ{pIbLyj|W~#7`wmzJUJq zwbl9>B_i9F9#v68<*Ba1?Yt_6n3vD@Dw1?ERRm!Y6Czp!n_cPZUlLK%smLpym7b~p z(ynHs<%lV2%y!6PKX*fZ8^cJ$K$P|}vi6QC!ZVqBRBmfL7X04MYcFxr%et=ZhUtdI z!_Rwyj&!yBy3V~PYNz*RA@Y>h?sfnA+EkEw3`P7ic`P(R6FF$eWN^O1_uSSiLpQmh z!JH0&MG`Nj^^C+8i?`7j5X}o(=KDEg89fvul(m}cS?|$N3Y53t`R_N6?E!H;Y2FJ3 zs)4DTzP^v*b>|g5xtTE^WOBb3g@@?Ig(eRV(=#HvDZ0_-eTO5Iv@P1f4#OQ!8%ZL) zH>wqZY|y279@#X1`xZPu_`NJl$h*}mb5iJTYIf8zvBI?3fq{3-ZxVim-CR~Ko%A_V zA#{8?$$W2n^_l0A-5P)n6*@d>-$>pGT$T~SAPWx4Op+@X#jNwS0W7JJN8E7Fl>@Af z#%3OC9gf%ECBdAr9?$X}cl$iNR~2I4f@ee@b`BHEt@kKVbt|+qTI#F5vVl^x9lG!D zZx2_$RA%LUr3+LxXD1^n{Pvg4)kRJ&^+&wSryvYe82q@!>cmQ!VC9h!dMPE^<|HZW z#_xFntP?_RGa8NuVvT-#ROl?~5NEhHfmJ4VFAuUA0GKji2V^QWr-Y2$S#^5DRXVFK5A`R2iA&lT52SPb4 zC0|#3%d5sGK}<#JxTFB@ocomJa6mcb_DaXT^?9^=o)z^f4m)XL;O@`wWRd>6I%#b83W-S$ZrC z-kqx5vmZMPRSJtsT^Z=;(hUnsVM1c0Fwc_Co`O10Pi(UE*RzOUw+nIU#!c96YP0dE zgOL?|JFeh*6!P~0)!^SQ0#zrTwVogR$urpXkc6*4D!SFc*tqPmVSKjR5f2L$!(+i) zS~;%wBQ>)hHk!4IO90GkIWOy4DdoIq0VA0^M>Su>%`SQM=6I;wMN+6j+4x}D} zNXMhmH7P`VI}XUd^;%|}mFpkF7N)*uPg`8{%<1pV&T}siJ|>M4Q(?mJT%CR*L<>a! zjYdN%$B04BcY#1+U4s1D6RNTC6t(=06(&AMs+-%IGqw8=RzvnAQWq#t0qS9iZ_%#s zl%vNu+MA@|T&XBT_DyQzW~=e8{&B`D)T-&4vF6ptd&;dUZL?^Wf!mjO2_eE`W z+ia!wXJ%(HN-GHJb^@_J67pnK z*Y5xO4|PiT`P%(DVsBaF2G06b8};BJuGpNfTXk6WjL#jJ1br1>oDT-Wg5iQcQIbD) zj$+n)*gHjxmkGwhPn1X#@VnflS7)~+NliKU$LC*Srd=5ic@wRlNJ~PX?85ALL1K&{NBn+rQJ{jaz@7q0T__jKR?zYq~j?fn;Hhe3V86pKE>z=mb zQ?={_Pl*Yt4|T{9ImwfWL%0q%y@z24F&{DyRtIW^CkkQ=rh1hECV%yG)KnU93NZ96 zq_KEGcJ_8d-X7=Ll>#Qvo)d4j0OJkbb*fkz@PV3Aba}o920M>zaP1G zD<7TsBt#t*!;K)UMVq;>xZE;>W2V}6pS&VE+VhOFR9Ed{XnMcQIbN%*A2MbJH852T zRV(n@(|P9N<&@tKW1f^*on})_ZG7d;uuJG5F;}IivSy$83A3!j&Pg$^8|mJPCUwjg z{4Ns$lc|0=G%&KReY38rO(OnL5Y1Yy5!Tl8ivb~t%<%A|#3xhbiHr^o7slff1pxsa z!>Pvoc?5)~UGxKj);P)Bvk$!|W zf)C6eYX3wBQYs1!7pc9$E{a5S%`VP|ce=U~32u-_c1tUr$FEpB>j;5;B%eaRum7#D z2rDNy#RXL_{?Vtsyu9n%vQr6XydPj2(o=<615IiO>XF|@!_HqC4DLiH+Z2JdJ>RhPAhg~e-)Z7gvd^r! zt2cUjnbnxw=?%SJT1u49)m8XNQj;uYlFEe9-+!?Ajx{rLCju9%HB0e3bmp7saAjiY zaZ_%vnslJVSN_uq+U)m}X=D}G@3epBG%aD8*2O=4zdW%yx}_(8n*6ak#++8}chc6q zRcK{c7GXD+-gvhmVyJ&mL9w~KoDRoXyP$yB+M4q_SFNEe@;x8vb9i4_g$$L=?cLF9 zMcZrcbtfV~-*&IH9>z0SP}!YE)mj-L&AUr~xB|LZ=rk%9!M|iS4>}5{B_=d9Tpy33 zx=v5oTt=t*_`Ke>dh1BqwdmuAIPId#e;>2CJl?socrOS$I5|87NEWCnr%Vf&ur+Dx z`t;eg;rNS3`%Cml{Y=Pg%8te0@Xt$r()ChmNaSTaY@s2VGOK2^v^}c!yFV--p4q45rOLr1r zkC5s`6`xqJVSRXXpB1plR+oBkzi+=DNnk!~sCMd}NNXX~R8(O*+$p-o3OW_9$g#5| z(c;XK8&emPvn6HFX?=NlsyiQuCb?bFL+u$gl3YE3{jSf9{CIOrj-iTTt&M%*%qyv; z^G)Mw?*%W8I(o^dZ+FIxm7|)&vbQ#HqB`9oeuB9dk692Y&$tuAAf#I zl*2@^Ls@gtF`ay~2Pv-30?6Pi$BflrGt=PJX2$xb*MRFOa?_7VuI` zG*-r6``P|WrP*Hr6=Vqk>;9SLrANfm3tB^ky|NQfnE3F=Xe?ywhXMIC%OZ;)ZGwaI z>%g)ymt`V*)Skz?)}AMP%riq>AF*%EBfBNZk)`}ijLE%0K3_5z8=Lvw*M;P!v@7HlK>6Y6m)x`AmI_gsLPcGwN7l>hWGI zNW)wO!We;+cZ_k)Px24kd~BDFzc0kMBvZZzAH(f8zvN5#-BU`TYKrAfgHm*zh$Vwb zk~RM=&}<4{r(CYIJZ9lxQ5fK3sYLM6mH25maBlh@g3p+lJ1men3K|G-<2Azi%C5=9 zPcDv567R6jlB@2YXbM$|m&W>defA=Y^nI=mQwD|He8@jw@3@t@j7uGRD$%MmI`8%e z4Ifzzls9uFC°4m<5WF+R^}9?oSCKXC8)+%+cWOZ4R>$OLaeLVJ=-U*^=*>afJ% z6KFYe4VKi>F|}Rg^%yQazJ7hw26nkKV*-)imF5RH zI{!@Wkb~>)1{9F>%n!!$FpWsQusBn$hW4qZL%IwGUwzu_UK>4n8bx44nH$KfMuu7t znmitT6}Fcqh-* zvZfVtN80Z!B}xoq7Z$1RBG2R>)Wx31@X7jw-ZS*lJg%b~%1!4oF^cRh2y?%0;-a#* zMsgVq`s>!$bFCjq`i6ql80?&RGhDgyM5ye(LJ~)IfugR&!JrShc<(_7f-9bN(59Yg zhR1sG+8PZ{S@Iks)8j)FiTNP7Q$-Nwk`O*jCrf85S1})br{pig>FeIaZ>%PO7&r9C z5Lf2rMU4c!Qk~|CDv?5L zjjf+c(=gB>aX2$!?PLn3wp+YupWD;YFt}Rj))(6)OHi-m02^UNB~5`&LNBwve(y!A z2v@WA7vE2zeOfW$4i%z-Fr5&4dk1A65^K9t6dbAD185^r+~kHIS$}I=cM0QV-~yk5 zu`fp{*9Y{JR8M&*%*Oi;Yu2RS*>5j?1DZgR8~m?T_utwdG3z^WzBz@|-xXqLP;YD0 ztGZshLh>G~cr{K{i2YH-j(eTi+dVvKbK1Gz3l+bsoi*;X)$t4IRGBCP`M7@TM)Zpw z_l7+navp>1?Y_zHP2e@h3_e3<_HPy@UX_cy#Ih4N*H?q<`=Jr8GHu>hrMsIuhv&#X zBc}MFTA+!fEXcXg;X?qlxBZ2CJ*nBskpTt*M< zloz>xqbqXW*eD-d3Kh2dW8VQTN8Fkm(c{X$z(9oHv1Uj1<-FNj(3ngUNkjGXgde*h zMu>4$%Fjs^aRF5tRh16|q{QMz&+xwxEjXGbKX9Ppl<7X-(H6;$%e+uo*1De77YR0}qq3=O>?M_n{Fx2B9FX61O^*u;6`Ag;?DHE?isJOryYf2YpchOw+yXheZ`gPJQ~Mn z;6R1x+RU{B3u_bp)R2vcjhuNR9dsoA>Qn5TSXWc`Gdu8!gfn}-pg~H{-##-Sa)cXD zC=EG3h5gPs^v~BM^V~7xGv#v|ZX$bYvL{&-X;c)o)fGN$RdjIMAJ!dnZ zONpa4H*)xPQE{by;_Qc4?mRKV{5KAx<9l7F-u`O{Z%m}D{9~P3;aq&cSSCkBCiv}b zGyIdbaem}q88Ro>P>IQ<^ zAu@}W5Y|iYFl&qh0%&VB#;BV5tvQk(KFUzqHD+jwTwn0IpWu;JbkeSljwD3#OqXm~ zZ?zp>AF*?c78W$Rc93#?_1aIO+&(K1#)}tBb562smczn7=oJ;0J9fl*aZ~+U-=8bK z$7l9AolTr~;7!0tZ}~R>nP|~fRCzM;6=78fE=jo`>EqOFoi_gLOTcM#;A?yy(Pu=* z#-!_4B{EZ(Y&2sF96=1=s4;jMhY3tDO(Z$7M26pEpzv{gwsI#9G1e?lqzO?`ov%HC zx$A8GrRB%3!YCfIRAH|;8NgdbFXZlbi7M6Z6I^v~NSAbajN#0|5n6VRhwDN9dViDG zGcd&gRc|L|!ABY>^4eoC^-yRF<-2B`p8i3G3}0t>*GbY+GdQ&@N;N7f%H?!>i6VYj zBVKh@o~L~7X36R_G--I~m;QLxbW}z|r(@c8O-AS6(tmBj^)R}3osLdz%7XiM!d%34 zbpvf(Slp%0@ynKI>wJWk? zfTn>E;9$j3Czw{1E=4A<8#rK)&z;5%`X&lHC}8MQ$GTy~38%hE7+q3`9Q+%=o7EjB z4?>DB_<@%gaeaS|gt)6ueU@$rfhMW+E`~s7*qjn!=>PRZ%YSj~FJ%U&74qQmd_WF5 zOnf9XAqqk>Cpzl!>`$%*cD#Qy_}%U%v)+XhNP$nKUls4}L4~wtfimMD9b#xYT7NA# z;XEkCWDtkiXGKD%jpy(#<2|=d41P3*eWz;sO-B)EkuSt~Nid7rf`Q%ZQes0M%wxS2 zv(gBp2p~AnjlMHFynQq*-Rj>peTyX~wI{!hkZv4GF!+q+CTtZZL62aFYp7I_L!&F5 zTrfy`nL3bWp1|Rg{~O8jQae|kF(MW#DC^{aMKQdov(o2;0hW#R6r4UUL|$wU=f4r? zwsEfXytyQ`%%M-WHKfF)**OFkvLV~eRycA6mxnDpyFx;f0%zC-D1H-dokjad@r;P!==@UD-$LxovWBOBDP-A-D$j~TM4?8P)H zoRuHw4IVI<#U6^={DCd4)EM35XOJPf@YA~wq8~bA(6v2smLN4qMGnKqm|=@zu=r8Y z!;I=fYEF>QPu;-vpSWU=L-iF2)`RB$jrE-H6kCMo_Yn2|WaxZ>(Pb=-XhKoJNSKxj zq0Omcam=-=DR~3V29s)Re{Job!gmu5UE9Ii8-%{_I{irg$1Y?R47C0pC;gMA8+5Yl zgx@oHk&!Xber~~_WF6H9x;Dy`kuDauap8GJEloK^w{}&_5y`595)ToiE8oIcUx6p( z;?{|eyJU?|i`Q8RLJRs^DK5F7Bs9g^~lTtBBY5 zh!Cydu6Q>tu{HhatnyB*$`H~5IeO@NQsMPH{!YUKrVR{1bE{k?g~-3H1S#7==(dkJDAz79{D4QF^ZYI?yN+{YLfe%F4TTMersRTN%iCowDV zS2CxUt#5rShZ=+|a@KERQWHSa>t@rT!{xi0a97xyKanSC0a>w`I%FSxgLj)w#R~mj z8UI=#h4Cd>+GJot0P37FPj*J zjXQs7=eto1j*5gU9Xw9lYMco?^M##hdOr-lp0PJ3=8S#koXJ((*u0Kx%y0+|?>a{_ zAVKfEd?89|nW3XEW8{Gh%bXNcw7g+jqSCTRpPz%(c5%J7=exP_Hq#6Q~(_)-` zR7;{lAeK6+T@3L$ILJUGYD8^?>$0~E7W~ofZQ`c=H2;AIH@&n8-SO754qUxbRFV6dO>+#MVaSTKo}K;?&0!A_pSH@ZML)= z`w>!GAag;t^Tvl>-Ovt;yqK#STP|GI;NksT5tuX2Ueq#}a5q1R)H8+OY2P)0S$V;~ z@ucHk$`&t&f1&L_8+vUEb{?u=*(piZHK08NS}B=A&u^*6q2U*zo$4W?b89(tk5PH@tJ1UPqF*+ROJ{p{(u$XmlJ3OuUoUk=h zNP~YWhVz2UszSQgx3f7YQR(CbF`{P6-d<`jzp3kqm2v+w*w#crr)2V1u0&nY(}BmA zH&xg{UtMa&rYnyMf;wk{vwmORt0W@@I-+6f!jWAXo<+qnFm}A}`Em6u^eCuKEoyF~ zwVjOJ)3(W7Y*O+37(mT%!ooUaH!0PyHmy$n+om&;^8!Rb*IE{)qsXKQQqD?Z*2M-Q zX1tczZPTQpo5J(sbf|VDmi#jWDuiOzfYgb?pCB}+5pI7|Kb%6;WY=4L z429Yg(OBU@Xmz73H5QJcKRlP(a+7A?jbg1sWgjTlU$BIyaWGRcz@Td Wf3hUs|Ahv80ezK_7cUnz2>5>h^HGEV literal 0 HcmV?d00001 diff --git a/docs/assets/terraform-aws.png b/docs/assets/terraform-aws.png new file mode 100644 index 0000000000000000000000000000000000000000..06974642a2cf704833c4e71d0205f1a4f0346e6e GIT binary patch literal 37726 zcmeFZWmuG7_b(2JgmehfJ)op?cQYW3v_ncucMsh?gd&1SDcwCvcQ+_0(%taC$M5r8 z=YO8x^WwZe=Y`ib%)Rfu*ZQp3Yk$@rqtw+DaGsDqK|(^pQBsuEL_$IyMEt`*1HO5^ z01HP#qD4}YmD2Vy*)@LrQCnwbsUx4_xuN}TH@pK$LqoDc(!L9RYi}=C`ILwN|8Y_x zekZQIhBh{vEVg9R8uE(C*08^;^>WLF)qYMRkE`|4!y5XpWg@#=&*{0F-J%*PnqM-g zkw^rzD6&BqsWFmFgdjbPfBwP`jwYam=*AMP{C$XzOi#-Q=0KsB{Aauptt1?bQlM>e z+nUAHKb&45r0KNw7hn8aQV zy1n;pgJWt)jCrUwJ*;E%uj*uyK~NjANo|aYz3W>OoX_srS?LP!f1b&9fKV90*>7zP z12C!$YL#7#wuJv^7)Io=q;GWoGo%jCU|yPgJe~&b7BSjjljMv&->ic(kZ zkpqQs(2XGWMw|-nmJMG5No&--)v)k(vyVqSmPEY;R6LC_B3qfG1=nv+JJ^z~T6EJ6 zNdIdB8jM6P{~+CEW`0bfJ6dz~X58A(`Gxaq|FYbi3_^V01?V;|-bFU)sci>7@xbvv z?*>}{HU!0JCm&cwb1m&Vw|ln{GV`D@J;!g8N5ESF1;8UrbmYkdf$MYYsQy&YM_bNS z3jH;WH?S^ydBR5wqVL^zpqvocX-5_QKd{px1E>rP&?^Ee4@a}4hxPu01>@jgd9Xcx zp-5@95C3Svel53z()nMHVUa{Ff*6qa06J(nN9*!*eXy*o`nygLunK30JrSrd6rUKM z`GB7tD+A#j>Z6GDV-9CSSPqe4k4`@gdR*vt}9d7qWI6?D8%4dxFU>0 z*0;W-_)K8uC}83^3xZH(9UXy#%M%_*lhwQjm2EYB^oIL)gz~-qe`O~2fV=%N160R~ zKq#4b@)zhLb&QFyxThLyoV0(%97Y5lTb48fm{eR_Hdw*o>wFp;L;pV@HX@>~itS5UP*`5*9vU(c0BFdHq+v zvcF)U5L&T)WTUb24*u(VRpG$w+^39#<3YNmx=KG)eLpQbI9BNsP=&IU3;mUj3nB9w z7+&^F-3q(p8AF>Cfb2dZ%-M(tt&W4L=#`IOAE^S4T=f@NY(-cxvO%ebB~klalj_i< z`1%3>6@G9K!ky4^pxb!ry0S+R6z%kY`2-O#Re~Ue=q%i5o!ScxHfg8it22lfkOvS8 zKT?K+Y1Cmh`UFh=lyfM6`8RgHBbW^lQ;7jnZU55yV@FtMEQu%d zDLuzi|50;X6(1tRg7AQ+;*zWfM2M$KFpnb(J7AGZg&^oQO*7OqnQIS48D0~^YzR12 zhXjD^!)y{4yLm>D5TO+kZ8a7!NwRbhG?N(I*Dj00<6V~QodV6oR`!bkRHOZyS~HB~*FEtd{ejslzP@-H>Uc~%ltlFAikOZ9rNF+9;0ep$qYS(J zgVWq02s@Nb0%ByvhGW7h`P_=ram?Y5ntxZS^JVK7E@G`Uk)T%XDv?^a9n;YqNi^4U$N zT_-~`8DaVXaX^$HF@l?e3T0sXiX}Rwj6Lq(7!aQ6`3`tnJE+5{ojobr@mkty#6)91 zn{n{=wK5`15OPkUL3~tefz(MWvadO`PYXa5DT7p~CwLP3?8sMhHjyhO=*Ll-86X+QM8Mj_4O$u%Aw!hgtcwh=8WvmM6MI_K19lh0;V4Y6N2QU)rkck*NU~( zEd^KChDf~`&PfYrS;8lfAYz7GOR--%0Ur928G-QxAdJTVi4Gs}XYm7py=czKufIMJ zdZu!<7v3atO~`S<4wPW&&LG5C7Py;2t7mBzC_LpN+hgNd-E?9e`cZ1}qE>nYGA56N zE(Q?QIf{VHI9XgvV7E!rKM;~{toK#(aC;D1$-k*xW&u2@0AV1j6t1OATI|Q< z%xeOB{rofrt_RPZ^=t-iv)(Q<4*I>uq5rp8coC_8?5dSqbD@THImgkVGi1u;s4K-k zEUG~GC81SGq~(@B0P^BrkeT2X(PGV2LmSh{sy5bkxQt*!MXNe0;JM$wp1WCh+p>6A z=BRv4(@|xdFlU5VkwZlG0I&P`di)9I1jollMc8t)s^DXXu}{7G1#CI}G=TuH)UFST~((tPp1&M5+7=KQCaq;NOuxEoc#t(wv7>(gzw z*%$7ZZ4;nmsQud)Qb5}I4kJMrXh;AUxKf~>w7XOiPC3_B2gHor3%$A+EJGEXV9SB% zjS-2AC;QL8sJ-X3n=VuN8jRhp3LoU^NY7%17@&+hgd<$dj|t5AWbm1Vr)t5tK_Sm+ z0eAZGG83Ynsop~**uUm|hA|-reX6nhE$-N$T3?AVQX_>cR8r_p97t{EiYaB5*sVih zQCv&j@29GRYi#sbLyMkzO!SUUtF$33l;avBMnDE(1e4NvP0|;Ol;#z*TfNhZ zk{%{v9#(FzWUG;c52E}QZ(@X8BYkao_1ABKT~D!+JoKm++0p#OEEEy&XlfD@#Z*`8vHjH&Zh9J+*+&7YKD zQa>|`0du7%0k(Y%IPkMq+A@v5hT{sHCti3UkC@p#+|?@&i{G@iDBD&zR&N!PN~6b!Yr&7fIQHh@sC7ZUYo906@Jg#bt;L+eF5F61El;_ z0YG^XfHDrQ9}RvvhFs|euzEkg^i2Ww+xm~~^ngY=|GfXl%dyd2XU<`Syb$_2zgwrk zWn*%o_Xp`f`qS(~{2$r@_g6;x)mFx!roI7wCD;`mKY*N2(%%gdj5lIb6q$UGCOe)Y z>0+9~mGb@hf)kLPRQ_gb!a^$1>a>?EyJSAQiVD#GW$T0J4GIgzhpyvaBCKrnMzA4Z zx_+^uXckBbqnRE`lpzBEyvwyekVSjAAXuZk3OL3uB-9Mw=TSe&*C#CW7qSM*jAyJ- zvl)`xBml@Ca{_1>2TxzJ)raA&=-WiPE-$WPUl0#(D#0>SSoZ}EyecXui?iGGMuDwt z{r;vsDT>J2(y@ciF%z0e180Ju(K5qqnBxqsB2!-30)>Xwch;$T2TSaxt4&Ue zkC6)o;QJHv=ZRd8QYTs(cTNLbqrg}8mO-tXo_dUfZ`QZ!$(8f8iTY}Y6=9k3vuZbt zU=hK>$tU{)zn<~BcF(7_vHr+Vf^{m1rP;EC=>myRllFfl!r!H7N^9v9JbTd_JO2Kr z7&J*-p&7?m=KRGKTF@A5O2Jn6#ig_hAj7{vu4Y4CbUJT!XuB2mS4~$)&RznNLk1Qm zkJliui242(1;Rp_Nw&ahUG{2!r}rcHsi+M=@Nm{KW-&lO$jKXefLkIDsk;J!kG%6m zf6+nwu9rUiqgMw5f$ta|s6HR+{2fXWx6_dDF zjX0)Ff#`^+fukwGo=T?@aVB!PN@IUU?HeGyYo{Kphkr*YY4<%#1f_&arr*t!t$qzxOWMwV8@MqmP!t*BJIrN;O%Fk9pZ!lM zLRUJBZVtt;)_V@%I)N!rvRgC2D-(ne!rYxI!cRjD;sYLOIkdN7$>`vO{nzJ#M1`?Z z|4oJC^=vL}8M^^Oj*ZEifV` z4oW7viLlYd1Ow&-o~`$-YMbSM)r=&>>G(7K6HK1VE_np%2mo)klc3w^wTXTycuzF| zB>o4iVY>jVvE)s>%m@D!_OMfmf?40A#;z*Ax$B%vIshe&M8GOJ2(bcufPF%W{l|*m z&6#ZqJB!}&crsV0sXanmW&qR1Txum)fI8Nd0EU#*-q7>huY=cMz-BEVCO!CK)|r7s@)->iWgP;GJ-L^5c|-1{nK9sObGIWFpBLMvLX z9yjU#9S6Pu1}Poc|7M+TCuRIzH#y?}hWITAEg1aYovGW2kp6Hh)@)73uKK?s&ljM@ zf%5K&&-s{~E;kww`Tql)HG!w}LF?!OCikZEWTA>cusPg4CMeI;z9id?I zF#o_GmI~rcn!Nk&seAc>7IQ+ttxW;|KYaKif)Ttj`uG~KjCuoBb+ubb9$+6+p8vpa zO`r{5vYYm8&dDYy_Y^n{Vg0EQr-u|i&Mi++g0;QDx@91=`k6@T5rH9<$?KR&H*7&o zXvNJk=N|!VCnFXHKxzrqJYS)MSec?d77DRo{e)PC`hP4ljbBs*Krul+sU#o^2a~_B z<4orIDwR1Ft_ULw%ZOu+k<@>Ft_bU+e|sQCSSZGp;t&PRWYe7UPk?3`-_A9o1KBU0 zyiOYwXRLc%56qu|@gI5OQdORy<8G5DfsGq66IqxrWbJ>*5lkC&GsMX^WK!??Cv9Qn z?UZuLSrd-MPw!<+U^-;LPyd}+j^#jP};6$g6$XwE!6w(MHmNn2Z9(BiI54cvXBsG%isBYgoSvcRfoVL zSRboZnt`;wx2l9Q!SSK5Mjj4E`@GvO10QU|77XD94vvMGv_SQSbp19Cy8Wnz%}*Lp zeeI5OphT?-?47H>{;KgETWxtLdnS43+k%Y8ihur3Var!-h$ClsJk2IK%JSuq5rv_6 zTm&@J+h(d*OJ`Wr6@7cLQEpSj9B&$NTn$ae*?5AY#p~g!_`vi$jbI?Hatkg9Y-DND z%^ukUOc}fMwRWVIbqsZmQ~+V)t~mGR8*NbYP?UKFQ1Oxv_A%Q4M`|URwDB0<-LcnE zF4l8a*~diA1(PhMxWs_sAKz71e?-`0Nnj`?v%MUrX{Ti`0RYmCt9|vu@yF1t1qayI z$MHy*JnwqvAfp&%Cbd%34d%gpssmrlfVrQVr|weF4p`@kX=e$PXYMllaYg_dZdSY# zwTM;6s;!}(VRu1M!eW4|_Cbas)iYx0qu4Krg&FKF7_zxDL zl}S`g=nyeV!wRZ1H>w;IntA&|A8!fKy)Y|H$15}gB!rG1^ zaKiP=557^gc#?HIr@Z2@eiFerSoOIl)#IwAYJx@tlgA%>PCA_+vRro@piA9^YroYv zrhE{Q*cZqRu+ZoX15yGJ=i5b%)o;nq_o{9m<#z2W!SqpcjTfXomK~d@W<6CJFfQrq z6Z@modY$I)S@-7N$NOPu<9Ion)uOY-y=|c{15XJSUjWbm^bYg(pD@Sk)7k5a@*nmB zaW-Ohg@%s-i@AiYi=jN7`d+5{HJNgzG2Tt|v!j74dn&M1f(fjCEA1#S)zGAH$*|7D zGhYMDF+H*tJ7ws(qYeZ6J2*fSk7A}w$&OziVVw0a4ywo0bAdA0AWd&;7C$!;TKms^ zlJnU}gB##{R!59`V&1N3oet2zqigj-!IC8aS=;0gAv~@$RI?47ZHONqWkdR(X}^U= zVT)l9K=_h!E}e(BrGPTVK$Oym8{?7*Tob2G>@`^@*5<)&oOXpPYD>^g79PzKCJk%& z0^MQ-&Th4*Sbxk3t)!wva#B3EYB-z)%8W-vcm%eYJ@f7#3Mv#zDgeQ(O)XL=YB# zr)*2cq0foLMa8bZRxs-M@pawFx@FT^alhO(OkwK0sAqTBJ6T}uOPI=3>+i9 z37mtT(&6($n${}Q%$&v>HbC2*Kn={AWlQ_;0pNetlMqQQi%+FI$B_1CRZ0s$BWKR6 zuKf&_tOL%+?e5tI-OpLrfD=*}SLS%U#Df&TY+{o;pIcJ}1LQUcO6&`qC)Jvvv$;Ic zs*dyag|h?ha5t^_v)`4iXSF}jq9hT7d!zU`u2}U@>K_Huq5Rf`qx(r^Y^aC<~hVYn=#*DD1Zwzp+>0~{gmL+y9K17*ds`xcu<@=^~J_A2S~xstT@PPNZRQqXi36 z2z*^NR4fYTww=|?pfmB<6yVBY~#yS^5rG;qOyCLtpA9rpYuISrRLsYfJSS%Mzk znol+C2|=t;1x)+_Ip&DbU|LGGKN!$VE@R%1vPiQ~glW#)2RU760F3qrA{ok%f~9G} z7XcIU(XG)hJ{8Eo${67^Gh7Hitaih)a0F7t^34#?sYuqnQ?23KsMa~&&5u$3JbR)7 zQ?&8yV`^61o8Qh%z_OTg$?}c__7=RbZn_?Nbs#Ox3H;N?&@iB5d5ix1{QAuJnQQ%a z1lO;-{_E7+K5P*Hu?m$2q)oj@=yo(8(Z+X8I&k{Riy#^4q!~5LSsfz4G9cxFDihpB6bWTmKSxH8tmI|3f?k$fCe^>7pjcs zU1(4&(PjTB*I7&yj17z%L@Ki9ehrv6>ZneGfa$eTl>mL>0s<*RaY|!C;KKkKCRwOt z4nRtRQIy)Gfd;|9BMd;Si@rvO;PdK-OEdxuPC)j<{<=3-K+R(^$MK2Jx8$Y&D6<#Om_7O0I7cb z2zU_C8TuyBR+$0CTP!x*AM?3;#% z0>&1tVw=qb3c%js(%*`juqaISf!HdZ&mwv#RzGVW0#yEDsRy_a}gV?n?O@ z1}tgiWuafeLkF3EDhBX9AMDvh0lmI641~PL#GLr{8(_GA*2XU&T*;{QNp`^iQz!T8 z|B3j2jACBnP1C>c0{lNj|8McjJ_J8G-eAdX`+Gd3t7G$r5{!S$!VrQ;OF-d$X|y?H z%se7mr5KP?*VK`u=yd(hR{skqr^rm^NOlJUvg+hN=&t4n)Hl3&sc60WPXP)6N*6M# zZIa7HP@JLTU=3?}Ip;e@&1`W@9=?`Kk6%EHvr+1ku7jQ1=Pv55j(1M^9t#CZq<{eW zByNQm1xPB=i7vq-&Sxb!#~oByUiF#l!oWQPOkz`3MC%mTZ{hC`Q%7|B67N;K>h(Jb zfmFCZf?;ifXbyi%43R-{&L-;N*jln8OJ;_UEGD2b=P&}ZG6t{O{gOtaV|j;hefa+A zuPj*D9MFXb>IF#y4R4-QvyFmeIp8s^TDPM-0dU4|_PI341Dd{Uyiu-gm;{%H@6*Bs z$(o-_!`JaJ2^3?2nvXRULl*x{v=}-DAt;a$*wj+&f-o4vkN`SVRb-<&O{tZpE<+Ip zNnzPC$3nJ+)g@Yp*NT4twrGKttbiaCn7cAZX zOcu`8ayCT990ABDlygZ!kXUOccfBkuaY|ha+?Vm7?+^ihL#BW8IJiO&VA>4yLH3ML z4E68WdFn!+-{51|A$T*corubv%V-kuWdg92u65*3Fsl!z8Le|D0j<{Gts2U92%|J+nE|J8 z<4JoiM&Jz@dPOycKmr)AV|xv$Abt#+1l*i|1}}M0*>K}lx_}{KX>3Z z04v5Ody!nkVZ8cO@O+dGpi}}MUZfx8r}x*Jeg&%4kc%QFNVucEvvfLwY(PmK0XNQ*HJ|jEfFzyIcxs3ZT;3Rb*PB$$@&?r&;V~qKW~-9}hy&7N zX;kTups%)ou!(FKYTt!{VA@HnlA=!(19$>%n?>$5)tw7E$hBPjN*KV7Sj{pGUIBgP zh84ffB)K+ebs#buEd;k20ewKogr;;TSJ6Sa2;KeC_6Qla{Be$KvZ=Zk+vPZ8{H!|3{nw=)ds*WZ0sh5WU*adrO%?u3ePzX= zn9NWWu;I`tB~fDXk9g%l5Tu3Q>l#BMH%Ap2wmkFIkonVd#b_)tP~QqwVK1qegk%a($*! zz4Ea@L((}Y{|`qtf_t_XFOtdmN|b6#rV4!c?pl5Ligl_B?~UE>4oWnaZcU8O7yX%q z4hFS8H{CZ6DSTl}S@to#AHUKrF%pvgQ_0v0`R>C646S!`nDeVvwagXs9GVA3J?d|! z37~h~yk%C&_?q~_YH+M4wzjB**KYcC-MowIob!!?Jv`0D5NHf`#CmR5SvKiT`@VDK z>h^lcjIV)3bg`?!m(Pfb5EaVY*4sQ1-Jr}&dyL-Kj`od^4Rhpz7c zK?D_J+SLeTnY=L(?hSY(!6@DUn3RgEPbMn4r2DQ;b zmU`dH+IK)2pD_ZzvB_O~>Xf|UDJPy|T*H{KPdDo&rkHASPcq$^=@XuM9WxY2n(ZlB z(!AdiY80sRYObj(y{fN!s6cJcKo==fATV#y@8<4!Vx&E52tOjk>nlk_2>p~tr4ql* zxFUy8dqliBR&^BGl_c6k;FanJn&arf_adh0?54%Ao%nChqzFRpalMgmy5lJNKqa3| zosZ_m3y$RV=`N@2YAViPQQR(rJHpc&RZ^C=Xwf+K40Lpo~mn$ zIlPt%8l`-g=JS2XAL-4|YD){8dZPJRIfL_hHj3jO@TsWWBj9rqNZ$~XC?u{a{q0gm~W`WjLJQPkCQ3im^I{P)H(@TN{F z*0;4IAGi;^ks!EKTBxzo8LoJUMbPYUU4422J;W ze!5~J!Kyg{X%3% zH&8D3cPGY9hqoC!){))Iv8L`Xn|3GFKPidFz)!^_SA*RBZWz|HOtvVn9_Jnc8wz&0 zJ!{U~?cd+}yWWJI}p7wapsfdfyg%pY$x0inBfqVaj7N{wZY5e9CnTZ(QYy;o5zK11YG z=27Q`TN31L@(C;t2Q>XbtAvTkp?|fz8}$Uyroyo=I19TJmhbX_nO#K zNZ{5iD}49Xs{zl^e>C9uN2upB9#8d=viAElt>G3~tJ{~XE~I;>Es)Ssi12(xv*PAp zCi8+rbyy2uKoRf6)s^?VqDznYSKR9xV?Gs^Y616Wc;>A|I@RH1p>{`+v9jp+Lc4O% z{nhQVjo%oARG$|4etmwuxM3bA{irLiN>O#quzuF)e8C#8@o;ydpi9>_Z8Vog{1ggZw6#8&iRL*1-uXaK*4K-)+T7ATGg)ds8!*_J|#&)ILi`~Y&%hM zSx_6B;u1z#nq^o>)4LJeb3G6_Ml`tOEaon$B!3f(x_K+`ieGDD*SEaAEFwleliF74 zCPm^(bKX-}i=5B?BO?Kyk`Ftk#*o@1vUTNhy(9aYg1A#oho=8Wowk$v9+xY=`o821 zgCx1Win^>l2(t4~8!mxQn7;J;VIS2u+Z`*+jNn^;vLAmukmEd}mQU8K->S6U1k{<* z%r$PD7puh`YylTy2SxFdhv+yo9gg?g?$(=W3J1V~3cqcrBo1fay|H!S|A0Z@!%t#Q zF^ooV=jnM8MPup>n)}_oKDJAb^TW`k5_xsM2ol=VL?@6MP$IVXbE0vh-5aaGE+NM9 zkHCsTq(D_vcV?tHX7y{aVA6rdC47s%J!kw~i52XLeeMDmNh5{eG;iT$gIe447QJcE zK^s?Kr}Uim!2~FdnvFuhDeI`RiE43rOQZFG`t5!eM48i_(}-Rhb}pCBX$sjdR>1KG zZQN{I+JYn=s&>{zra1Jzj@yOe>n1i3+Q| zzs{(>jSGLomZIYOfk|v+4>Xj9|At>}qqFlTG-V|1mBhn5OgLq1Fg@fx25R0YsTuXvEE5GVrZY2d(6S ziIPLpJu1PGW4KRv%vgPw%Bt*Y!1t3U^Y5j_mO#?S9otx0AW<5D1$%R{>s#}2(XJa# zdn@Z4&(AFnzrJV=cr6PL(;j7)W_%poaye7)T6$BjCDM<%m(UM08a-^lbW%#d`lcDz zWD+G{P46I))Q!4_MO|$yQkS8n_J?-CVSzs1{P9&Qms?FNwSdz1gf^bdUgfk^D*sQ% z5shTrf~$w2*y#$?KJQ{NTUt8d*uI^XGXqe8>t|{O}97r+3>E66~<6gavMrp z$}NoVRAVde0)Jl}5Q?ci1mPfOQ|C}0Vm-7DLt_MK1U(fsS2cEgh*muX33RmLie1uP zr+o8dh-sCHWwK=6wM=vVB~d$o>9Y2~q1NY1vRsHKai0?#c;|3+Qr@mQ8&ulK#6Jh& z*f|TRe_c0gPgIS$Ja_Xa)-fo<)HnNdrg^$h+3%ZBIopXx?9WLsE>6=opNIYGo?(|N zLaV|9qhD{`=5*@&6>?R_yn3FZk@gBDF~2zMx_rlccbnq>*?-}CN;iS?dp*;@48QA* z8IX`7VqO+1~Yo&S-L*_kq9v#6%vFA{eTCWD7hjc+sVk$iYmf#WbcJR>6 z4;tEEInCObCyAj3N(mEc$!qbL)8;}C9tF1)FEM!@EH-_A%rbH`U+bsyp+8)O%g`!e2IaHZYwWJNL4P&1n&)xQbk>_`S!X`XN?u2)@9Y`w(WA}XH3^c zzJ{jfXD=TgPQmvzT>%E;&3Sh?y!oec{-~kOrLmj3a21tzqtt2R@mU$=&3ra!EM!O3 zfFC04+KFsth!f_eN~Oc?&KdTomr3s|JwqX03#U35GbwWBtumJQIWEr8r$ges`K#A) zmaVfF!b+PmZ)&mr|%XAa3E79 z8$Z_5tT9lgLFTX#HkHKVFEduR{SpbXx!>9A;P3^jHU1XP`VyiG-DC9cFeR7nokPTBpL zk;)}-jBjPsJWE^=Ty9)H4Q21hLuPh|a?q+^EpFB?`KYXsHEQIeb*nf7CGP|mvHSJt z5Ju-xZopBkBliGrq+gwn-*CE9+s&B$w}zOKVhU33J6bJOdU(u5lY7J#Y*lv1WMKW> z)MdEnh2o%?sodeun*{MYO@W`RvE9GiOP=o-+J_C)NGxUt9No{R?k~L{Pqfqo1gry6-?T2@{X!e+I^hF#ff4)gwF)n+)IsblErZqm0 z$F9$A^*lL?j?j|uHRxks9NDs|$4aeb*xj#eeGy~;4 zR#|0wJOK|{L(JC_{xl9hGn6t(Gl!ZY>)FZ(tlf9$dqx=z^Kwr1fIBH|N^EgQ>2k1H zt~Vti{0*A5QnvzCyY^qmRQZ>@%(uO6$CTeqsrCy(EhZ=YD){WCA}#L6QhdnFU0RLw zScM=AxuqX+1&c>AMf9i=Op;9lbBpJ<$cGB@ZBFRtJe4o$#^07H5f)bA93Sj9R>k-j z;L=P2cQ=&e25vQ!ZhF_$cux3^V6!iXy!w3}a2pi`G&z@?B(g2UEOTX8b}V}<^rnh8 z_)-Kv)!AysZ|0w^Ycu zRhlFbk&qYW`c+iemW>koVtmh%2u#u!c(?2Ad|67)dQ-Z3S01H@=1sC)zVWBR{decr z_1Fnvm56f%D{Zu=A-$x3f9im1v(NvVaPuRnLi$_SuyI1xx6=abYErN+-`R(Eda9-# zk;2Vxe@3SZc&e}75c%9@N^PFrU#z%-(@FBhK$#h?pR>AuZb$nSTr3B)I!5CjF`e8B z?Ks&iVT^-^Ly8IO_)YKr5uv+rGwz0Bk5?m8A~F5csj?~z6!B^r`O1Rdj| zMC}K>jA(oq5}-Icc_s1FBEN5x=dg=l{9s!OMKpX9{N#^9 zNEFACk35DDxt_tf|K~LfsOdt{nR(){#1dHAQYymNm$B39F7CqekLLt-#M;NGP~>QW z4y?;X>M#8o4#}SO{fV&IGE0CP0Pg#3Y*zGr2;8n65Bxb$h!WaIoL|W872;UCVbyK# zzJsIpOpBGUU@d@K=heK$79P*RCpi^x+dPG1>&Uv7!|X@MkM(UqykAF-iH-~JW?#n3 z3X3y7ops2FR`uJjA0-)SIwZf6t0w=Y#>f9)ZDvenwH~hg{FF=Qw(pbRlsI=4%e?uF^2bj^}@5l<19aSEsbr zeSG|}CpL`n>3~dF=nKAf9ME?{ZcVxirm6_P!uJY2`Y@mdEu z_Nfhq&_Y9MNEVx<&nE@$5Xd8%4BQ^-%u`t2lKAN*QY)+9tHPZ-iv5@Q1vxOT_q^Wj z>+`G}snt!MN=@Y!rqFbcx6R)`eZrAKNpT|jaHFy)65Pc=8li_WttqYZNQ;QXao?2X zYxVHBXRM|HXK~40gk2T#(r`EDJ3ZEKl*PXoWza>_iK8SwR>DKW<72fupYzA-J|zI* z@+~+_NXI*Q78v$i8X4>0~BfyGc_bi2Sj;WqS|#_kki^B)`kxyGsYw_c$v| zh4f(`^7AZEH8Of{c|!80UCMHje&9oC-lNgi$~``2_Kpof`DEo>bRfJ%SM%{y^T`U;Xa7x&VLg>Z{6rnJjF>l=9=gf9D`k z%Jw)cMSs#_zlQukY0sj##J=B4syg5}shMZ9FXxH8pFj@4oPB>j!)+a=I{p01#-gEB z|9aMovQh2fZSOe0JcAg+^XJo)J8uIu;6~P~+;|e}NZSFn%qhl9`b3l=zek!+g}rq` z_XTVxxyTc;-i+d!C{`HDdCWVDkbQl=l)T5RkDZA00MyD_IIdyTB(~Y#r7h~1yumM> z7Lz(Dad0RhLX`#?=H1=K@Xb;mS|Tde7~ zJBH>I#hX8SB{$pYyyd5=8FZLsl>{Gx>6!Jg@x|EwMmKZy99#VQ#clgd755PxN+yE( zd~Px4*pK1Icjg=1cX`6u{*t0n>xk~pQ! zB@i`qY>Qgnn&Nw^_md@TzQ6>0XpR4(qV%f;zkDARE*C$|&j3jI&J?4F+CgWMSp}HJ z#`R-x&w&*BF6$Rt?l6Wa%)(Dw37LU?ss{DMAXl%3bn#JdEnY`110ndk{nk4Jn=fi| z+vy>tZ~bmB;6Qb&KYKjVI2Xw0o<+HGMoA=V@{zcE0nY|X@5Z6~8n zu>`M8_Fum)rEDL-KtWJd@qb2#7$qd7p|TflCVP0)(b@Kk+RM}rn#vH=-fJGCDBJOD z8e5ib(Y*l*4np>`jlK55@VwKuwhXsvq|0b%wQ|`{-ZTa zq7SgkX7rmywg{@Kb|de-z2gr)iSk68UZ2j>xCjbE@HpBkB8&nz9@2_7?aRgp&lM*Ta z`mxoWm9SKqJ?ZiXzoyG7yYKbTerT+EpXj6T*uwLUQ=Lk7FZUD2NTX>O zKod#_B>sd1ApX52k2#GGUccj0`-BL@X?$bpYs{A&~ zM)EC}1xJ)}yK8-790Zgci8NDuTKRaf1ck=B(ETFZOe|NE@vfsScQ~*~R?XMtMI}`VeRr(r9Spg(^<-E>H zo5O0nzBb=IE4h(GoB9iC&6O5!)+pM&ujNw43blp9*U*!MYpGqN<13ni*etK#A*Y zkwx`I{39hMem(2lP~2I={rx+Mlq|dz_w8k^1F=AsPIPnUcAl=-v>J~^M+za6>ibjC z^6aNhzfI5H1Rs)WQ$@A+xLd3j^n~(|48qZC=M80?<7XD@_EJDqQBG}~BRJ%sgTT{F z%u44MAAe9=v|Q5C30HAaE!4&31RO_IB_xAr=o6nKx+c=;d)LxP1~cZb@Fsn$JU4D~ z1L7CWyR7P!WMOLx*j$auTHfIIT_yr7LV=bw+yb=*Wsg)Q*D)NNI^j&+KO$>_x5XRk zMelMGA2COsJF?KN}{D?G!%^f40k#t%^{XxLKG&B36o57Pyer6-bqsYVCr;&2pBe(Ynh}o7=C? z0==kA!{IpU{j49jme|yX#j1S|u!fpbx`|DFEl1qC;0IBXnSkLM1znyXL*+i2Y_V=q z+Zix&yyP#g*IRqXH+W>g|7814(;GZ~l3|*rVGerI;!0qruY1KIKutt`h_7lNpq)Kk z?56X_c+pf`zzc>=B`zN@%D8;SN8@ON&dfZuq~U=3(w(t&N{4zz^qc4M4XikrOa-P9 zl3S_-_nfH-CQ`stF5y=3Cc0K3Ql_1I|4b8_gCT>?KQXh_R->7>@dNt!S~F}l)g3C ze%JTUwy083ny=2$!IZ^2UTdubs8&TKl>=%E13UR>WM-;c-Z!m%St1^>E%_H;xvuD! z$5bZ2%~7Hwr?yKw_mbuY${Or7TlX)Se2l^Z8jH&#%Qaq1E@PWGItu%~8WDsaeh9kl zepFgEoqoYzhlN%kiWz{f40vW9vE9AelWcJ~$#v=Z*{>ecXL!B9uS*k~nw=!Pi&k7W z1MnZi2I#?SgP-P~x`cgjGxOc{8F$7Ny7UUjKKcR*Zl6z$<97Z?u9nP_?!3SFc>J@L zueUByhVO%a14sgW?7PbJ^GM1#4fk-ZK4{5q!`(EuS+7?2%l?Bxwl>NyoQ>3d9NUwM1pS zy=u_ogM7cg*-J~(v@aeGJBYvnnwkykWYKdiH}^fq>#j1|*;j%f!R7ZxpG3gz*|A1kgHnwezY_K~`Tb(;^5JsY>xDV5td%`bnm7{i-+H*Ii8fw`6xyW~VJyU3dB zF>IOTd*QkkxpuZ5$2`Q8rHGUjpumph+10fL6HY1?=hA8~YPTB9vE;t`_(X|*F5~vS zt2h^4jZ1;9tjU~VdAq2?knmLH$=y4|?Yw%w6BiE}@V)l6n8@tp0F(!Pznsvw;>B~> z7K;fwa6y*HS=oR^@G|K)4VS~|q*Gh>+1Qu3Z#5byDDUA!pqqr;t@jt=$?;!|-;6f} z^S*xu_M9LkN=N=ANu`Frxsj1WHG%U);G$_$*L4KT)sffO^c0uK$`t4+IPHxey>_Rb zpBhnDZJSN?pyWX7k%WARE5n^&Ebf|Bi4R50}?ekieD-~&%jYIaQcB5kET z?}B5hV*W*upoM+iT=%%k&0fLHb1KU(=?i3(x75#W2ZL9jh@Tn=Ze-0=`@B+N5*vGX zoDTuf56Y@ZY~vUD=0>{NNAjJLM8z|K(RGP!mL0R%RnaJ7M@D!wTt9nATs%n`>Sedy zWCBAiY1YV7wQ%9l zZz*KKko8~ASP{QM@fKfQFWyTK?HH(36LtEc*}TrHC|e)i zwShz>HBV;baSzivkSar2vf`jSe0<@ft?j4OsOdK2;Xuz&`3uo}-*bcD{Ehg9p7GE& zzp2lbbl``KsCj%zhds3erA$MDk+3hyj(VpnVZXJlh2 zkAPNLYA9doV1AeAJ;lyMP~pW&L0)qLx#yF6;?J^StKQ+ee2D-XYcpPs`F zxwTlMCZ~2qqA*+^X$wuwaiyOd@U0ixajFcUKCv0mayVlW(_YM1)1NANn#pbQS;ooC z&O&(SrAHR6DW&H0@9QxBl@=SVIfoglsfotstQHFL0|QE<#8X#}|8U2W1MBZ&+_%1>fF9wIHc+T1Mq(XW2BDP&JmnH76Z)kiX-`N`a(5zMDgY*)) z!9&@OzYD?5qx2zq-n#0+zgvep-t7k0#s%Ui`@DvA>>8KBcKlT01q-6UZYcuhLLBNPL%! zo&?u_O#MNJ4}xT?KRx--ZB_eP2_F~eTJbY%ijO(87qH&RL`~okk`}ern{!oOMWYC(TwMX`Sk5+R|VzS>yp?q1ARrFt&I~LEP zhZsa&y|stq312B+c7p2w*Nw(_HpW5$kkDC!~xs$;rF{i z?UN;Rz4B0x9Q=g0f>AM80j7q#>*Z9!ht94Ce|Y8#CoA6Q-W`20-N=b~U|tD$NNJ4t zktN;Y_=$Nek4|^EgN!7_OtnMZ9-~?j&jIIjbW9e{grd|HKe=s^1x!4-AN^Hq*skPFxY>L zi>ZZh%TS;tUk!RXTn5OR{hXr|Ta947=d?8-;(*VNq`GP2dplaEiJ7{^i}xLma2;}A z;!56u@b44JgGB-mExlZ}=8Qpj0d;w9_HKW-jq$%H8w2X_p3qZd=E^YfuBlR+^*&TQ zC9^cESZfMFJS35{m2J{GUhd(Hnyrud*TLkAAQNWRhAyT(N!>^}Q261cuZh%$KmF)v z-XJcD8la&zwa$LYff?6oX={wTb)o%~u{2>Y&6(7!Y69;^18mgFuUZ%Bn#Av`OGZCi z&vKa~yackZ5AKW(dR%Y5f)n%m$j|eb4{p!OyP2&=TV-MO;zz~%6V6Yb7@R8oE(aE( zHLj9qp;EOGW?{ze`5p+f-rRg+&IykIug5|o}=VfO1^xxM)8G0=xzC2(2|WtGL%t4 z8UI9;7v-sn&HvRZw|SOVt`7sX*1oq!oU*{!C&%@-L~3N4;M)EmKSLNfBUF0GMc2#V z=MFag*7K=*f|SyN&eS8f(V3m3BVHB*zReT{mXbCdvjW#LT+Etk-`9}VGzQrr6=eG% zWHLdn_K-X{yrD_}|ACGQ$}c++VYi7r*&>ne#dB;8!iK!sc}b_n6&~wt$rfb0!U{iQ)@bUnrS*`Rj(QEvMWqwQAoIBp5>X9UTRIqCp zg7rfp&7{Cc0Xxw#EVewx9Kh%}SVR$Rjt>UtqA{m%2Bv**erydcxbqSdqIMj9xM8Gy zHF*IZ!;I;ED}FP(up6CIV@sBcA7`cM?s>T+Z(4T}RnRc;t+;r6ks9y$SqmCI>}PP5J9sBWfL$?IZy;ZD&w0E?*cZ5N(e*#qQ~ofx;L zTqTr2OdBPmFOFMpWbYdbZ|ski7@`eL3k0naBb&EtHWcHZRc(Aa!I@jOrGUo*l^~r8 z_2V(7&`%gy>CzHJ|P=+`k4JAOi_xj$)vnNY6Y$VHoi4-W;3hOX@yRQSblw6+qGwh zWS)kpWFfI*aEga}l^Y4;p=7F!&D4~95KD)vYe9xK!4C|Lzx}GZ3_2D2yk;*}NYE+8 z12OVX%SIyi(J08o9^RXFG`mA(o5l9r`QC)a(t_crSwL7UKReC~ze05TugnM>60(xTW!LD7&p zhOYfuEW&7#*M$w#4hFPooynFdj*s8V@DL@CyHHOSXT>eQsI3|w3O6!7O$e#Sx7zIQO34;C%P0i z7CZ%kG$QPJML*0Wuf#+z7EOuTipGGSHuwahhpD-(`Z+47Zc<1AYobG%2!511EcR2J z+s%z)h~MekVL(`|E?sH|mNU@dpcj#bS|kkx%Lq{Xtmt@5)0&(6%qht%Jyh@_RR?X> z(c1yatnYLm(6CUn8UE((U%Cw1y(fIzAEw~!b&AvEs8VGL7NW@71fvFfA;Fii)*4nl zsPO`1>nOmoQ3Tv7Fuaove;jXtm|7kaqs8 zK2T{oVzHtxdBTfewsKbj(jW*64Szy7>ON_*(91fVD|#zisUbF2P1w2Je>O^#Ufq>% zmj#PJVZY~gu4FWMsS14?X-cWGeNU@J$T#5;kn%U<&V5Vr83(VEYpXf`)FK!>1& zl!QOAU&upx8*$#;7-e2z^4LCf(3L1FezAhPP?j%?Ok0Uuguj8cP9c0Xo?3I-7ca=+ ziS#AS4J7hxI=%5?5gIi3xAHyG4pR+#gba{BYMGNd7aSKvO}LpdPRzNszvF{VWeR-i zM#Nc&=(_Lp$eb`{MOP`l|KvJbP1h{T1UybPN3#Lm-^}=lBP}>VjTH8uKY-nb!b~R` zVCxiR@Ber!&erHd+p!=mdN?a~`p2Qd*JTXuk?LBWGC36Y!YqN=r{SCrvh!Pd z=tH}_&D~bbU0}jD$5u?dFlJ}ULmt86GhH6St`C=eb0zWV#^Hzka!O5S4H54IS=-*--B4u;k>2GgQT{wv1 z&K-y0P2c*1Q{!3r+Yj?;=oGXX6J!WiQlK`xdcn4g%mfZ}j@Wj;I7?ncnbb!qa=xf2 z<=G=*4`G`I1^u?wGqEgiWqdRsr*~~7+Ma`JHfbfZZkMhmAEB^rH_0Keh z?cte+Vd1@WX4#kpxTO-Vn9pOEWHX@+Wr5S$h9T#Ij0Hjy^78W_p^skOK$nS&X! z{JP_zCu_MHCgr#czmiFS9gj=%@l?WF-mlPzFo5Y@>JDr4j3i~;ejwKI2kXq=*x*J! zImIXKtXKrjgnQJxwkgysCBz|oTi@csRp0unZYJU6Xw99n@MdmlN5yuXXc$9qfgR7z z+Vzts!7_w{$A2F7~&9- zO4T1Lj&a|5_XOE>-~*Ht(MnENTu<~nKee<%hCMAr><5_mh6j1`fB^^^OG}Qu*UN*p zz0s;7^RvA*iE{~dOzM;SkS!mxKGTe8<<_j38l@v>9^RO02+5~3^Kz4R8m1joi+XB` zaelctw)B~2xB1E_WRDV~dj4S;x+5=B=uuF>V~8t?Y})2G57mf!uhVl{ne|7CcLjn7<$L4-yrzDT(7;<^MvS@E@X zKib%!+00u`&Oaq!Hd-=!R$?!Oy$2T-Tx-pH2F0{O$={x=Xp_oakx2q)CtBzF4cX+R zM_OFx-#K(At}Ev0$+%l4lg+2SKBcr2a% z^c`eX;prs0-pu2B5%(%W_6N$Kc=J^uUVD3-*5g#EUlCZi{xbZJ4$pQshg9Pk_=6b4 zTU`#6pF^r*-aV?h8m-gf5jF-j5OR@*@6#@Q;<;H;2Iws)i}RX=G|T^R_`O1_eDrzr3Cj?$(evlZ?nTodN*&faG3lJ zc3AULhHd-HbqMwpJGyvwDTwVyV>~rct1z2(wE#OLWa=#-z^$e{q7F~8)u+3ZXOf?dDT4cj!bdmEHFOmuA zwac(<^nZG)tALs@9>?16L^XgEr!b-`EqxMc=sVgr+|C`CQRon*=P61{FU&tAWR4Xj z@jYWeN@4_>6L{pN@&UReqU{IN|uR?7!LN|CHD*OS#t>f!o$I+ z;n}ozR0U;^soYvwF$F)hNk4W)gtSS)wVW<|>`z&7V0M9v);F>wxJ^N3uDA5jvGuSP z)jn@^&RrLT+tVm2oXKu09+D`0+csIg7lYDk6(C!pbT81w{!&;L{~T3Bo{%H4pH&P2 zqvSoiqJi@c;@JCbKSs~@(8u9-?8lTU3r5ri$l&J$p{WwO#9mmK=q@IG;cHN|y6h0qIKjyJbVTatzf^S&`(jIm}gw zj-;m5#z3Cus|>1JMN*#PFSEyUxBHg6FbfPOBO&m!1Ff?NL&L(}Ha}th2=R-fn@EU; z6K|b*JjNWtqRfu3f)Vc)prc7Sev#xf;ZXMc9L7~y6m`6MJ;$t_VBPUdMG=S@u?XF2 z#h5?Fg3=?Hb3=i5OvF3KKztQrf&n)KJ=l{(wer^1`D88@Quc11#hp8~@anN?<1Koy zM`i)1>lyYbHn$I6QJ$K1jn-zax)Y#aJ5q~IOcC}%=uN~*H4g@!g)|>MOk4MY+q;i$ zD>W-)5GVLL^35siE~oCb6v;-%vE#o<6V#jMi-Li?`GdzBg)>U-3okd6l^t;JJ<*jF z5`Jv@Y56+UWcP4u&?qw>=>D)2-GnGInA0&4A8PrSZh4nan29R zqI5>_^ko!nXYypvZ2t61$`Nh5$~NLv4gUhv;x_j=y$D*5#Wj!(k9EEOQfui?=SPie zDO~EmM)EiiL{hW0~j}5O=dY?Dt{sejPOkuBqW-BuyYq+zX>{-f;oRKT>U%Z_caLrmUjknaMQna zS8)2|!5Ss5XOc%tZX5k6#Y27kgnS4vQF9# zomq${>vq5Y)vPcBL;AP=NG6YSBh*L&~h*8JHavpgtLTvw3?!MLF zu1({?wjl{b6LuF&;-csQz`)*`!leAYG==FEb>~62LQDzGs+B8OFExf=E3BNHBp%Mn z*lUyfQoEq3)MsTI>VFIJ1-E9O>HC1pb}VC_(h3(!@EASIx~3osZig`u$B&@Y3=#~A z{)fzkb$(CTyJlNqi1z~CNfzKS#pk9E0Y|H%{w6uY+Z|qGZK2;F7tqfB%=S{=Ud@Xw zYi1RLZf^H23jdn{x_CcEZu>py|F{5DJJcwgZoa~`Q!7@eQ{nt3l0~!hEXEbtyn`52 z3u4zR&Y_y*En+BmIF|zVk%Y|e($DWjS<-kS)K0xqKuC<;-|o0@^wC$6)b(+vEXD?; zyxr%@&cz<0j!?ekL?93<)%PitG1=WlXBTXe@lyT(g=&Y`Jr9EAM{`guT0`B%jJ*X z=gikzeD_n0+R_!GYnlU52>CFv;^R}$ERk5|H~)BRlqr>+IZvz4yxmPWtdDz#=V7tA zjZIdV?DFOdAO^a3H5}yf7TnsGVu=$5G5Nb!HXuCn6=V5saYlU6P|M@ciZ_1iezMnH zE7-N4QDr8*{@*)gGKH|6Ez0x+^)rd>GGr`vn5bQ5p{d+Fdsg91u4EHYbc^YIC|jVX zm!!4ZFF&rSCpsquAtCBCAmOu2NtblkZ*B?-b2Hu7^R|< z)WFE_f13$ca;8TjHstZiIrd})L|=JHH*f6T#qGs_31hbt4D-*~Ule>QB&T=LCFKNtplV9Py`*r)7u|Pw`k-a2hSomi(m49lXKc>#2KVde zVk!W#5QHwfIK16Q2{=og;)=XOd ze_^Uis@V0?a~;H%)=J&VH-jt4>+7i5`IRil$%Hb;{K}PiW3@4 zienLhE)*CFA!n14d7T^2EbT((tWH>SJw!_VAvW7%{M3UZUyMW9VP|@Lc3P9x_E^R6 zsY>i@ql`3tHya0# z>FqoUH=`-ZjGJOw9@nOOD%Pdf8MYCgHCy#kW?|6v@GpK}brN-b(r_KC?d-mv6MQ-y zN=&mgD2LOZ3^KAjKkxus{%Ix2fqAySFzmCu3U`xhbhNi<)9P&sSdp3#9Z`z-pIA$k zlOSau=813CXOB8wWosbE6`1$WA9)OzM!Rp(b!r$848Ke_`%sdwo7Dr;bJOD5a>z?K zE}*h}?k>X32g$``quUzxz8Z+?>r`NZXxsFKt;k%gGLy3yK{R`!%B87{Eyh`XW2y6& z5sGrRCScE}zr3~UY?|tQcux^>L-DFQ-qL)}XS9ZqQC*hImO__|XnauOM0>o>(tj@S z>=9e!D`_dSM` zKmphLHIMPq0S`k z17b&>w_Mfi@x&6e#+>@gBQn(8ktaXD`={%3m5W*W&iAY@v2 z=bYDdQ(ZA@-I`U&a1F?A9E)w7nM3W8zM@~r(f}*%@YGx{Y&=H4fq9aO__%E8s<_@M zHDoB2P*F>2;>JVdH7CjRmV`@1K@Jv5uPEl)w<>BiI+a+}3>?fz_Ayt^^{-7(SzQYL zzB;vSQHAYd=`}j{Z{dEo+p~IT_FRo-NNHwPk*@S}lkd1Thu`a4X+?;h*8|TPQ&_C-tCFibqYT#3)FW77= zRw`zCG2NhXGKIh+O%}$Q>QUWrcj;=oFw6%Z{<#DN-DJ2RLg19|k)KBKF7mT8IV~BW z8xl&h+AM}~n{VNfEdqsS%?i^ebW12rtHY!LA$%xk_GJQYuLG2Yz=IqXPZ7Oe*soPm zxY$r?EwtO8f$y{Z{^MOzL_{Jo!a=E+|LN$h$++JLe#dO~9W&+OZF2C1g|JuTH?-TG zjzdq1!-!-`$rs1ALXhVj=y3j*TwngVlt^+&qVFr7Kfum5tU9C2Lz_g?y2f10$AYBq z{EG8s&nM0@5iECkIcnQ!5Rx_GtLg8L)!pLN~__tTx zBiZX>Un<5)lsH~8*}FFDWGHKt4xPnVBQN|SC@L}v?K5FlqV@L5ZejPGa_e_qS^hpA zAArPgEp`q*C+O=1IPl1?TUXaB26_&&+Xt?&rE3dxRz(m*$9%?XzCE2e7m8EsX;F1- zm)(_4PS#XEP6dNvJ6bXkl~wjLu)I!$kp(os|>klT(1$T z3rf12IwVdojO1Hexcb!q-D}k{8B?pUh*VDMw&tn375-vPu>gUqJkQ;~!fG)RxoS)6 zdEOA#0&U~H)p?NN1ZbN2yy<3H+kSoVQYbWjvtI^~F%&Nhp;`9q?&c1gPs0&GR-`kq ze+K@?*($}3YRNA1h+jl3+(-H!sf&%o)-B-C>C8$lW}Haod#Su2t;hp`QsI&cJ0;AX z>bzE7(9|4DAkydbAXmX}Q=jb)1A+U6fcTBP_exhEi9#L822r=uUXi19uWa7r&v>mg zmpS`|51X#C>Un#Qb=YTb=6`KVwfj9tTPcX8-V>Z0HT7ARA!RqHhfWF0antq>dxl=p zPRP67-it^9;-@Y=hS!^vhGDl1&Na@#VD^l!39~oYcT+`rW8*mY%3hq8PG8Q3>G#Y= zx_q72>o$Vp??4EYSh{weVqLKgUcqGU^D+IraVY@+QgT zS7{|S>S6UZhc%^#R<8)Rn(x)mO~=QKJ%rIk+{7fNwA+6JWZ=x6A3$^!o9u2!@VHOC zv=%>S3AR;JftwvG68GU#BzE9^7~Zf?WLvk98!8=`vDoA_x|5nTnvyh+ zO!oviGg&5jWLk_FI}bFqQ={qs?mwYLv#uCu$#OfASysB8Fgc+i9s3iMa&)y#AVZLj zw0-Q+pGvpz+I&hL%o@Cu3kO*)ta|Gdf+Bf2{t^R(wkiK^q>+D$gmAP@3xAkaYido$ z^q1KmP9*`XeUx3ar_PHg$~!mtY}$AV_V_FGJ-a!P;#kUdniX0v*8f>*B(qr~!+$|V zY-!*iBwbCa7a;VDb)9bvwEpTl{|=D)1HT6HmY8Qq>tswF=rsE{v2DPC+#+1uN^P_y+x7R5Cj|I;jv&{8S z_-?{W^o}+Who{{rPKC+~XB`TeS*q0OMuBARvvo0*)8>wpLUgp2fSTQE7`hC8Lss)| zvM|2wh6~BhSubsED(fQ9(^-(W7Dg32htP1nBK#QFbwk|Me>k+2nR2AXiF_D9Ow6Mc zWfidub)S9hGK9HbuXgFBebke=*e#Sc(_1@0Ut*UD#;7oH8$Df*P^;=%C>F|pdG6*S z80AB{TjdxncWUFeJcJ?|UPxSDin|^anyj~orO{s2Uu51)Z-9ko5$3LIe-4?@v2PQ& z{OcFP>8AuxrH;|_>jb&C;>DAviZ!iTQV3kf&{9Y5txh$n+e%4RxAO+mU|zp@oJots z(CO#Ke+3LN4D~W|#G$}DwymQu%NW>fXwx^^Z2laacJ~n9PaZ%jdhLCEpcww>%f4<6b!D%+pFGdAY|3*M z!N`es0K3cJFf3S6zpxvDJN>8d<0j4G^f2Xxg*qS-qwlCg@kbQH_8;f?ApG>gdF2%X z2LUnFEwYZd0Jhg_Q{59%8u@R4ojw5Y;r_zClOP|oC*FBiKc^&n4B%sUA$Aj?Mtahf za}c*#P|+&72I2}Q#SEkz*(pzJ{5(g+@`RuxJsT#D=3Y{FOt>RzEwtzIxZ2n-Pcj>& z7_{R4k1is~Rd+Y?`s+M{=R>Q!&%3o4MT%fOlf?1(up_-{tvrYCAIiZQeELox=id?yy!8|&@`H6@laWU z%!31mEK}Jsm(cO3Q$sh-(0MKbRC`CUqocZ5|5i!KKmts*YPEM&&w?B>&@%2gIKt^*E{F*bg%H_|;$5|22F^E#Sv4UgkJe(3f?pv$- z^@lg%V?RFcy@L*QM+r-@KKO2B)f)kIv^_ZmOzlQwaM<_pH-*E(20@8J1tTXB)T`!rCfNrDlf^dMwu$84+G~g`fBML1ur!96c^(mKB` zUifjtfOXk}$#H0K6O4)i)~CBZQZQP&B7@A5r%$r=_5IMi7c*R=$&tC$WERi(TJ%i% zNv-yhCAr?K?xP>i5Ui>_VVH$1_$>-GxI&u0A3iIHwJifikmU6M-AD(m zGZ!@S`jFHZyG1K<=PSidv7~~DvvfQK^89n4KT19y-PkIAS|n1g`6Po4ClMzWK2P57 zy@5<#dmOw+ZfwBZlO7B7^HiA3j{`aJ9el<2X)`|&;G`qSAJqg3Oy z+7wVc;O$G~^HLe#3<+mn@m^QP6Tp)wTF62O@Zfr>qA3p^`Ax(^xvJRm0{Wt*qMvO0 z(@BG=*{EN6Z#-?l8>M!AK@%h|PGEKM?HO!nN4AxG>Klv8YyyfR0nBH{mr4hccU zisDSRI!`-=SHjiB*-U~|jD18UuziDY@qq5d9-6nLP~bV0fXcwnXm*#b=@9nRoPMML z-U9JmazjGO?^f~jsQzC_w^(O;=yiI%u$S<|6Psx5B>Z9aB1+3 zu_B6wCYeVtRMdoQF49dI)cigCm2iJqZ>N@kM>c*(hdZ&6{_}5k$nt-HjZAX9yxlK~yxo32ybMbUE1!2=IL#78P+Xvu zS5eKK+c_qYd9`RRdtXuiQ2)?U52>qF*(+ZV$hYvOoXLG1D25Q1>7^6MaU20ZwD>?j zbRr46LDZPdtgp|ep?R!gp@nHx?Y@z=u=6Fmn?0K?%4Hv$8z~SVWo^@^Ty=^FllojN ze+n2V(ghs+IvhI#{4awPfpN|01?b!fN{J&EBMR3F`y4MsMLWL;+N`Kuc9xbTt9xo| zJ}v2`t@{~81Hms9X2mqxgmUwvHOk4oeX+{)IAwA6Q2GAO=~457`?2$7({aLg0aCsvENIjCb$c5uq6d?;GUQ_(tT=)$q4 zIpvn0`by<>Pz#I}R74C!H^=V~XVeeFOGQ$zOJ1{Blt8=ivJ~TniqBRa?B{Y3Ay_?X zvttUV=Z|-`T6~ZS+K7v$H(l{#mhbF9EL$n)T{oR^)0_pStih$qn8c*AngGe3}7P?yjj6h*to3LBR;>L^6 zQ_p7b)s6{YT@nYo$MJNCbftOsDAd>!nIR;ATdcd}mY#zZ$jmsJ`JvAz)o@>7_3sLr zQJ4*;z=W$oM^Z{F{%gOi&uTjMmhg^?%31BXaZi^&{Z(Qhik)zD_nJr-&w{#Vhsf%f zp(HSb6Wv**qyR7E&R#3B>J%$1Pb=5u-WWe_Yd;mDjrZ16`6)Bz=byb`m=zk36 z_j8l#fY{RZ?NR3L_YZ+~2xFbNsGq$Y5$aoEBF~YwGj);lH!;lOR5iA2yXnk__SmFqbRTyP zm)@1tykxUTD7y9Gd3hRCKd8H(sm1`IkOA(8X;Rr(ly*+C@KN3{LX*T8QXvp9Vq#m2 zn>qeT9r#bUe#doV4WVU93Q(lRP~orgYx}6w*yhx*JveIPWB8WY4G_%J3{{BRH}sr( ziB4l&dWrRh7VKfvS0(g8SNmz-%d{20#RaGM9lOiTC+UPYK2IiRVYBTMdgN(C?Z$Qw z;tRRGTK4k@ok$MJ;rXRJn0*ax5b@9W!@i#~ZwjKHN|1$BNqM zsQDzJ^6l5ktL4y$K|O5ZnK3)JUS!!VzH2qK$*$4r5;F=0@x$FmpxtAYz5|&FpiI$P zLz)_eD@A1DsQ^{R8?keIwr6_wX+}~o;vAFT)12aJo>q9tMU_&^5c_BNGOERj8el@J z`?jZUI`Uo=*kY`zJhMmx9hDg+5{LOsMK@#pY%-UbbXzG#`XwA99{R;CgtI@RJ+p)R5Pl?z>rBiR_&tnbLaH~87wItCx9mcAc0>IcpI;N%bdAGlIFDUNvgA*RUdr=&| zck)N)q1Ifk0MVnXblr6^B4U*HzBua1EEvesJjl^!ZDRIX^GkA?e(e3<$!ub zcwu!neO7{j@dm@_R^lD>;vfF{H#0sFuD0(I3Io5#y@yRIYqr0JqhqbsiLVz>95 zcug<_3`0M9(K7^m#xt@4aG!!n&!}tsiNv1S#_{r9%`-HnPdf0GGQ_`)(4Zn=SEA^@ zub^UgY!EM?JD0E*EK_y|xC})ZdacJ98 zNi=kyZS4}0*tG)Is7z7%gi(#^+u0=TRx*{TM5jqFMy`CTnj1ZZR*k6{Dgz2H<;pk*EJxsdB?DjahoPY zSFs`iH>Ps*oD-mA-V8U38uI0VSJ7RdKvd&NQUew?`Jv+=n22Q}rN7#FobsMXaQl>j znbWPY_sD$QQwjYy93N_L9aL){9hX*Jp(1aVLy9(m@X7jf(k`DgwvUqv58puz#mFRIUNN zvTv69^@DQzTPcq2c6K5Y(vk$t($8n%MMA|GJ-u z+bi^{Ot4tip#!g&h~sG373zK^jxK1_9hev037ZfcAc?*IrCoE8KDCjDxOD+6f<_FV zTv10r5bLZJ7e)xHXcx8pR%ukf{|Me(-Jd(pWH0)s+h0^zn_5ug-~4-o`x%8DJYikj ziBSegvT1_c&w8hJTQ8N~&~pSsGAeN^3qaJx}mR7kA1Si$G9#r%KE>Lj3_fxgj`2j0`l=XS=T97ADG-sLl;D}klzu62)D8cLT!L4MruB?P0r|_tp$`4Yd?D)c!pArp1h|e zU9VCFo6*mhW?rYp_Ml@_sR(2Lk_}Tui zoePt)GLG0UXQb1^D6XVPym1(7=A}@mT6Ah99KXS@7haU+$Iy^_s5aA*bz2Z6cB+*< z9#SswM^2WWk&jy#%WJzjkIp^yseL`$_m#$;LErtML-Q34=#;LUzX;6#NcbYCta#9a zoGwk$4Apy>4w!oQlxzn%g*(WXTHp47$XWq|BS*Gy<_k}@irgJrO@rpH(_YmJwWv~Y zbszdkFUEHRr#!4C5)8Fp>@8jqbi)%zh#HOZ1MOaHl`^x`JmAM%L&AsMl?efRGa#>N|iMYRmjRu#f>An*&W|PPDlM?$-%+A zGVfcXkP>Dm*%d%SxU&UDWJSOGPcw6PhcAw<5q*DyI78D#E7sIroqry-vAX#Y;E%hX z`Oi;;tK&P?yyoXWyFb*cRNUL+IHuVVu-xP4;u9$-Iv-GXJ!sCBo8sj1wF5SrOrpWt zi}`LQzCI?zJeuU<2Ql(C9~DK-bA8H3Q6`(Qc^0GD2|;T1T?SRL%d1E#&&suZq+N~; zRNtbUhZXexhzuSQMoH%2Gv7ERBIaNCqpV^f~pwKHB8K!MB)IZNa4GgI4?$BD zaQY{AeKzaA2BHF@7+|w1qg}0Xu;N^2!s>aF1YbMQJnz=M{+ONm;IsG4hx4kf8(vq@ z`krCe&kXS+HDg>7ShkV%9(yFk=5Ik^|31%8y{j+>e-+Ct+N|?P#$>A>8K^pbsq|NC ze^WvyzsXc3HKa~zfDS8e>TKi>*_l%|3d|nle5`szpLkV+?eOr5p*mNmWLA??SlnHr zFYBNM&aXgbrQ*^T_gD_36>_*PK~w0#pJKf(0ziMim>(j?{COQ#d5wme`p&WPgclLW zhr%GeI@{uTRg@)pKxFP6_AS3NnIY<$VqH!MxC*iAE$g5wUjwb}2n(&&Id$4R6iD&+ zs-ooIEi0`n=MM5cUjnmVRgD{b!zC}w<|lC%*T*!^hx~9Y^VOS@?mg4L^>iz39h$gp9_*GV&pc5v$j}w2dV421{5qv}kwb7(O;Yn)7SIH~$_p4yZ5s|;!Qv=T4 zj25O(Nd#yjrdbBm9LG)yHMEl+qz7M4nsyP`o&>*GcPzAH)qN1dk5h?_yuedGL+O2^ zD#v1*ZV(mHfcj?Z$LWcz{yTk*jqxV4zm7vsCbzn1vmp7ChimK|MC`;*bn!< zL4=?tHwfYl(y$0US&h)= z8~=#T7kr2aMOe}l(&Ao#ry>> z<>f>=g{=`75DEK4JYlw!I7W7(ageudW3$AjN$qP++A)mHbKO&g&4CE zCS$v`Qk;xZy%4fo>9OI>qfLtO#O|1jA+CcL@jbe7ZzO>od58#9&qJ| z`o+CoCa$_#YxjK%w(0GY{TPFxtKzu`1w}*~MPW*s)exp9TluOKpMo3JW={IQvy^EU zi)k5)`-bdVib6D@QEt3%Dbct-86>l;1DiU2v<1JVrqzVD09CGgyzL38SUvvjGEMTS z(M-ZK9qbeZS^#T1QA2S`YyJ0fimk3z%54HDkOe|{5ktvC4l7X*&=J*cNtkbk<4z(t zC2&^Y`l-cY*r44nrBlqo+G|alE?S9V>mD`ufVEhnZqJe0&=lXq1mCu`6FZ7fje8-Q z=*Fs?arC zOX332ZJKj7`X3e+Xj3yyys~)2*p6;cM37k z*HmK+d}X{H5-clNwXGvx$|?>Cfj2akKFDWMBmrYC@8fhkXnm2+*u@vIn@dXTo z>4Lw)nPNDP!SD1M0$3^c^O)udt(Bksv|detcI|C=jTEeozr7NU`4X*cD8CrP`QfG+ zjITJ5Z!?7b46&(JhYe}EePX#|N(<|wWY|XM_>6nB&0k`c>ixP&JD4|2tOe?op9xjx z%}zc%tjC|aY?z4aoNh(}XGG?2)=W`7Dvmbna4xZ*OxU(K5UVk7YY4NgO$)W$y+vuU zn!o!f9q1Tp51vUuo1Z(bWocN;<@R7ppUwPLD1#WF3^l{>%~%t@FN_T(oWT^t#QLSu z-Nm#|VWkX7{6Rd>?jd4awWQh|XHrr)USgsUVXTfgXW&N%#ScBzyjd3+B7MD{z4!a} zD6b&l&hZdKd=0^}b$V64{;aCI+B@+xr9)V0M#!0(SA}?4`(50;kByV?>uW@;pJj4H zY9fX>m(7PVpFtVh_)J z%m_)GC2BVf^!CC~Q{h*0jN2^JrP@?}M63P|XHl{5q@%lcFV6dx(ImS@3{Jm@8LZ#{ zPN^?jH~mUI3Rh9ekJ9V4?!pX1jZKgUHgn>NxY1tps#!VSMf*+6JrCZG=_f>Hfv2NN z1>KGvk!K_c%UQ8s{JDV5JQxbVjzO;JLVvq*wm1zxJm@BKTXia(HWwZIGoFLR*$!&F zCckMFbmem<$cgX6g91sfy7QPAo4Gl;V+CHe%N7Q{6%lpip5cs}_2Qng#HvoZd4!Yd z*_xBCt?bMlzpOqlHeY!^(M%4AOa8v2xU^gtk8JdJzNix2wq_Ty5gH{hXvlq6#sLvT z-~L^d{0+gq+En=OG&<|5U$+jye1Qy54BcKUl7pT}B{US>pid~~8j5hqOtb1Hh#E#> zIDf;!({tLXrz)N_AOC(@2dUleyS71hPUp?9{(?#PmHwL;d6q}_^u$+svDjP)@DZ6u z`M5$60c@ckfHiolWECpa8Mp^GLpRT6O`eRR5?lYqB~;1Nu&Z=80@R&_bnF#9p{7gp zt0$i$spx;1;VwDy7e2YisnD&rQ_%29Ak9_Jx7>Q1_jOsgAeT+mJ#9sO~EhON8IW4-enQwxb z@W^J$F%g|r=bW9!q=2-?t1DFQ3W^$F;Rp*uBIRDHL*8XfIezKc_p3)%1wD= zH&gui2~yqdhU}wuchKPZs{qSSR1#&3B_Hin-jiQDR{{>0J~+xH0cA?L|GiYExEmkm z3n&fB2E(gfGhvbGFzaq>9Wz?Zs$KxJ{pxLRJ^Q8GT1#dIOz}dbXe1P!FMZDVbJ4kD zc_fP+E+(LPSz$idITb^>h6MZ@-d`@J=>Tc|Hj)TqH;cUlos8-=;R5fRy#)n;6?4M$CMmjua?%D2GaQ2dBB;cC1))<(&x~zvM5NSlx0;W*JXra zoJ&@oIfU)2pQc%mFidU`z#I8p+A}#F)HV2=%~pRWkHW zKye{qHY?r%@Sv%cc~jDcuc7%KBmu;H;y;%u^&VRAMCPY0^ZmD2+Z-@ktizl0cB`Qi zs&cV~T(vvN)wu_TtxdXdK2m`|mVYkKTq%JR0cu*?$^+!IaI-YRdbF_kJX!na`5*HD zut+LK%!9Dg4S}bd;^3!I zhD~)#XYyJ|j#u~jjdvvZ@eP!K17-t->!+Rs2`D^e*DaKg+6eLDTL34fu|bi9c!|L% zP*ri_EKM>7p!oN+1{s4(5u0_aw>%mUOkX395LAzL7%v1yqYFpI_HTwO)I(z5v^$J&2>TuVs60kb8#xfO1b8QU`6C@yZ0G(tNt-9Asy z_+IkZ%4DWgav~HdVR!`rDDo>mzC zevgwP0SC-U5wl-p$|aB_N&qOge;jR;kd^1|fT)2_GzNdl##?lJ7C0BNbPY&rX_)Ej#`n|>K&l$Zg&(TKMS9%fln-}rKw~7}fY}(i`dP~= zfdn`qX(+Gl001hpt%@D*q`spH9v{4~QVbjNV-m`euEZfWr+`JTEJdl7A91X|r9s(b zTMJlY`PaIxB?w59aqP3+#lh+3X6I5PRCPjCWP26NP*r3R(M4Hy=D=`uW@uSQy|-(E zB;bHKL4x$tG`R!>qXi-I!1ZY+NQH)5?bV!NLt3Mln76Dt9 zrkAI|##&|3dF}T!smHpci0srUKcj#TbLvUeLfJ_T&?^0S7T?bY5Es6VI74!yF1^aN z$R;1|-u$sizyWh?a`U66tppUpq8Ly4MDTNmVg7Mj}VtYy%La?IdFhbmK}t(Z0CQ#G$}rO>D?|CABqq zUvg{`R|_6bV#mq+;zJ-c|uh zv38}Uj#PnjKt5DLV3K;4LfxuHn@~{;K+{;JdT+;%D6kV~CSVCLwkwSs;MY#stGUEF zCP3=U)Jf@jzsR(ffCJ{V?q0vSa!DWp!XyMHwwqR4Itm4|azqvw){0{(mqffZE`er} rfCFYT^_I8QCEyZp3AhBNOalK8opHB{=C4zm00000NkvXXu0mjf_nsa6 literal 0 HcmV?d00001 diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000000..31af4c6767 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,3 @@ +# Terraform AWS EKS module + +Moar content coming soon! diff --git a/docs/local.md b/docs/local.md new file mode 100644 index 0000000000..b50caf71a3 --- /dev/null +++ b/docs/local.md @@ -0,0 +1,20 @@ +# Local Development + +## Documentation Site + +In order to run the documentation site locally, you will need to have the following installed locally: + +- [Python 3.x](https://www.python.org/downloads/) +- [mkdocs](https://www.mkdocs.org/user-guide/installation/) +- The following pip packages for mkdocs (i.e. - `pip install ...`) + - `mkdocs-material` + - `mkdocs-include-markdown-plugin` + - `mkdocs-awesome-pages-plugin` + +To run the documentation site locally, run the following command from the root of the repository: + +```bash +mkdocs serve +``` + +Opening the documentation at the link posted in the terminal output (i.e. - [http://127.0.0.1:8000/terraform-aws-eks/](http://127.0.0.1:8000/terraform-aws-eks/)) diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000000..cc20c604b2 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,65 @@ +site_name: Terraform AWS EKS +docs_dir: docs/ +site_url: https://terraform-aws-modules/terraform-aws-eks/ +repo_name: terraform-aws-eks +repo_url: https://github.com/terraform-aws-modules/terraform-aws-eks + +theme: + name: material + logo: assets/terraform-aws.png + favicon: assets/logo.png + font: + text: ember + palette: + primary: deep purple + accent: indgo + icon: + repo: fontawesome/brands/github + admonition: + note: octicons/tag-16 + abstract: octicons/checklist-16 + info: octicons/info-16 + tip: octicons/squirrel-16 + success: octicons/check-16 + question: octicons/question-16 + warning: octicons/alert-16 + failure: octicons/x-circle-16 + danger: octicons/zap-16 + bug: octicons/bug-16 + example: octicons/beaker-16 + quote: octicons/quote-16 + features: + - navigation.tabs.sticky + highlightjs: true + hljs_languages: + - yaml + - json + +plugins: + - include-markdown + - search: + lang: + - en + - awesome-pages + +extra: + version: + provider: mike + +markdown_extensions: + - attr_list + - admonition + - codehilite + - footnotes + - md_in_html + - pymdownx.critic + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - toc: + permalink: true From f6e071cd99faa56b988b63051b22df260e929b03 Mon Sep 17 00:00:00 2001 From: ksbde Date: Thu, 27 Jun 2024 13:41:20 +0200 Subject: [PATCH 792/962] feat: Deny HTTP on Karpenter SQS policy (#3080) --- modules/karpenter/main.tf | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 8a3c9c1b71..4aee274d93 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -464,6 +464,27 @@ data "aws_iam_policy_document" "queue" { ] } } + statement { + sid = "DenyHTTP" + effect = "Deny" + actions = [ + "sqs:*" + ] + resources = [aws_sqs_queue.this[0].arn] + condition { + test = "StringEquals" + variable = "aws:SecureTransport" + values = [ + "false" + ] + } + principals { + type = "*" + identifiers = [ + "*" + ] + } + } } resource "aws_sqs_queue_policy" "this" { From 17448b4782b785403a395f96e1b5520e78f14529 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 27 Jun 2024 11:41:48 +0000 Subject: [PATCH 793/962] chore(release): version 20.15.0 [skip ci] ## [20.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.14.0...v20.15.0) (2024-06-27) ### Features * Deny HTTP on Karpenter SQS policy ([#3080](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3080)) ([f6e071c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f6e071cd99faa56b988b63051b22df260e929b03)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0b114602b..a98b95b368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.14.0...v20.15.0) (2024-06-27) + + +### Features + +* Deny HTTP on Karpenter SQS policy ([#3080](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3080)) ([f6e071c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f6e071cd99faa56b988b63051b22df260e929b03)) + ## [20.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.13.1...v20.14.0) (2024-06-13) From 1604c6cdc8cedcd47b7357c5068dc11d0ed1d7e5 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 2 Jul 2024 10:56:19 -0400 Subject: [PATCH 794/962] feat: Add support for custom IAM role policy (#3087) --- .pre-commit-config.yaml | 2 +- modules/eks-managed-node-group/README.md | 4 ++ modules/eks-managed-node-group/main.tf | 63 ++++++++++++++++++++ modules/eks-managed-node-group/variables.tf | 16 +++++ modules/fargate-profile/README.md | 4 ++ modules/fargate-profile/main.tf | 62 +++++++++++++++++++ modules/fargate-profile/variables.tf | 16 +++++ modules/karpenter/README.md | 1 + modules/karpenter/main.tf | 41 +++++++++++++ modules/karpenter/variables.tf | 6 ++ modules/self-managed-node-group/README.md | 4 ++ modules/self-managed-node-group/main.tf | 63 ++++++++++++++++++++ modules/self-managed-node-group/variables.tf | 16 +++++ node_groups.tf | 6 ++ tests/eks-managed-node-group/main.tf | 11 ++++ 15 files changed, 314 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 06efda4811..00b113536e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.91.0 + rev: v1.92.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 494b187914..05f1a0bea2 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -85,6 +85,7 @@ module "eks_managed_node_group" { | [aws_autoscaling_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_schedule) | resource | | [aws_eks_node_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | @@ -93,6 +94,7 @@ module "eks_managed_node_group" { | [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | | [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_ssm_parameter.ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | | [aws_subnets.efa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | @@ -121,6 +123,7 @@ module "eks_managed_node_group" { | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | | [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_iam\_role\_policy](#input\_create\_iam\_role\_policy) | Determines whether an IAM role policy is created or not | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | | [create\_placement\_group](#input\_create\_placement\_group) | Determines whether a placement group is created & used by the node group | `bool` | `false` | no | | [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | @@ -143,6 +146,7 @@ module "eks_managed_node_group" { | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `{}` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index dc63a0a7af..facd05a810 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -325,6 +325,7 @@ resource "aws_launch_template" "this" { # require permissions on create/destroy that depend on nodes depends_on = [ aws_iam_role_policy_attachment.this, + aws_iam_role_policy_attachment.additional, ] lifecycle { @@ -535,6 +536,68 @@ resource "aws_iam_role_policy_attachment" "additional" { role = aws_iam_role.this[0].name } +################################################################################ +# IAM Role Policy +################################################################################ + +locals { + create_iam_role_policy = local.create_iam_role && var.create_iam_role_policy && length(var.iam_role_policy_statements) > 0 +} + +data "aws_iam_policy_document" "role" { + count = local.create_iam_role_policy ? 1 : 0 + + dynamic "statement" { + for_each = var.iam_role_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +resource "aws_iam_role_policy" "this" { + count = local.create_iam_role_policy ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + policy = data.aws_iam_policy_document.role[0].json + role = aws_iam_role.this[0].id +} + ################################################################################ # Placement Group ################################################################################ diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index d62629627f..503f617789 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -523,6 +523,22 @@ variable "iam_role_tags" { default = {} } +################################################################################ +# IAM Role Policy +################################################################################ + +variable "create_iam_role_policy" { + description = "Determines whether an IAM role policy is created or not" + type = bool + default = true +} + +variable "iam_role_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + ################################################################################ # Autoscaling Group Schedule ################################################################################ diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 072c2f2e33..2396f353ab 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -47,10 +47,12 @@ No modules. |------|------| | [aws_eks_fargate_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_fargate_profile) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | @@ -62,6 +64,7 @@ No modules. | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `null` | no | | [create](#input\_create) | Determines whether to create Fargate profile or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_iam\_role\_policy](#input\_create\_iam\_role\_policy) | Determines whether an IAM role policy is created or not | `bool` | `true` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the Fargate profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | @@ -69,6 +72,7 @@ No modules. | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `""` | no | | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [name](#input\_name) | Name of the EKS Fargate Profile | `string` | `""` | no | diff --git a/modules/fargate-profile/main.tf b/modules/fargate-profile/main.tf index 1e2cf60024..ffbbfb8294 100644 --- a/modules/fargate-profile/main.tf +++ b/modules/fargate-profile/main.tf @@ -78,6 +78,68 @@ resource "aws_iam_role_policy_attachment" "additional" { role = aws_iam_role.this[0].name } +################################################################################ +# IAM Role Policy +################################################################################ + +locals { + create_iam_role_policy = local.create_iam_role && var.create_iam_role_policy && length(var.iam_role_policy_statements) > 0 +} + +data "aws_iam_policy_document" "role" { + count = local.create_iam_role_policy ? 1 : 0 + + dynamic "statement" { + for_each = var.iam_role_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +resource "aws_iam_role_policy" "this" { + count = local.create_iam_role_policy ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + policy = data.aws_iam_policy_document.role[0].json + role = aws_iam_role.this[0].id +} + ################################################################################ # Fargate Profile ################################################################################ diff --git a/modules/fargate-profile/variables.tf b/modules/fargate-profile/variables.tf index 75816b0af8..3e37b8c151 100644 --- a/modules/fargate-profile/variables.tf +++ b/modules/fargate-profile/variables.tf @@ -80,6 +80,22 @@ variable "iam_role_tags" { default = {} } +################################################################################ +# IAM Role Policy +################################################################################ + +variable "create_iam_role_policy" { + description = "Determines whether an IAM role policy is created or not" + type = bool + default = true +} + +variable "iam_role_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + ################################################################################ # Fargate Profile ################################################################################ diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index ee8cce3b12..ca1e7bdc0f 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -144,6 +144,7 @@ No modules. | [iam\_policy\_description](#input\_iam\_policy\_description) | IAM policy description | `string` | `"Karpenter controller IAM policy"` | no | | [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy | `string` | `"KarpenterController"` | no | | [iam\_policy\_path](#input\_iam\_policy\_path) | Path of the IAM policy | `string` | `"/"` | no | +| [iam\_policy\_statements](#input\_iam\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | | [iam\_policy\_use\_name\_prefix](#input\_iam\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`iam_policy_name`) is used as a prefix | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | IAM role description | `string` | `"Karpenter controller IAM role"` | no | | [iam\_role\_max\_session\_duration](#input\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 4aee274d93..fee75db43a 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -383,6 +383,47 @@ data "aws_iam_policy_document" "controller" { resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] actions = ["eks:DescribeCluster"] } + + dynamic "statement" { + for_each = var.iam_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } } resource "aws_iam_policy" "controller" { diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 87238c8389..96d898b245 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -92,6 +92,12 @@ variable "iam_policy_description" { default = "Karpenter controller IAM policy" } +variable "iam_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + variable "iam_role_policies" { description = "Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format" type = map(string) diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 681a322b05..05834cc935 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -66,6 +66,7 @@ module "self_managed_node_group" { | [aws_eks_access_entry.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | | [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | @@ -74,6 +75,7 @@ module "self_managed_node_group" { | [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | | [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_ssm_parameter.ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | | [aws_subnets.efa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | @@ -106,6 +108,7 @@ module "self_managed_node_group" { | [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the node group | `bool` | `true` | no | | [create\_autoscaling\_group](#input\_create\_autoscaling\_group) | Determines whether to create autoscaling group or not | `bool` | `true` | no | | [create\_iam\_instance\_profile](#input\_create\_iam\_instance\_profile) | Determines whether an IAM instance profile is created or to use an existing IAM instance profile | `bool` | `true` | no | +| [create\_iam\_role\_policy](#input\_create\_iam\_role\_policy) | Determines whether an IAM role policy is created or not | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create launch template or not | `bool` | `true` | no | | [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | | [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | @@ -134,6 +137,7 @@ module "self_managed_node_group" { | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [initial\_lifecycle\_hooks](#input\_initial\_lifecycle\_hooks) | One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource | `list(map(string))` | `[]` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index e980453930..ea1bd8c4ce 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -470,6 +470,7 @@ resource "aws_launch_template" "this" { # require permissions on create/destroy that depend on nodes depends_on = [ aws_iam_role_policy_attachment.this, + aws_iam_role_policy_attachment.additional, ] lifecycle { @@ -861,6 +862,68 @@ resource "aws_iam_instance_profile" "this" { } } +################################################################################ +# IAM Role Policy +################################################################################ + +locals { + create_iam_role_policy = local.create_iam_instance_profile && var.create_iam_role_policy && length(var.iam_role_policy_statements) > 0 +} + +data "aws_iam_policy_document" "role" { + count = local.create_iam_role_policy ? 1 : 0 + + dynamic "statement" { + for_each = var.iam_role_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +resource "aws_iam_role_policy" "this" { + count = local.create_iam_role_policy ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + policy = data.aws_iam_policy_document.role[0].json + role = aws_iam_role.this[0].id +} + ################################################################################ # Placement Group ################################################################################ diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index c4767479cd..a3a207050f 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -645,6 +645,22 @@ variable "iam_role_tags" { default = {} } +################################################################################ +# IAM Role Policy +################################################################################ + +variable "create_iam_role_policy" { + description = "Determines whether an IAM role policy is created or not" + type = bool + default = true +} + +variable "iam_role_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + ################################################################################ # Access Entry ################################################################################ diff --git a/node_groups.tf b/node_groups.tf index 7228931071..2118c6aaab 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -283,6 +283,8 @@ module "fargate_profile" { # To better understand why this `lookup()` logic is required, see: # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.fargate_profile_defaults, "iam_role_additional_policies", {})) + create_iam_role_policy = try(each.value.create_iam_role_policy, var.fargate_profile_defaults.create_iam_role_policy, true) + iam_role_policy_statements = try(each.value.iam_role_policy_statements, var.fargate_profile_defaults.iam_role_policy_statements, []) tags = merge(var.tags, try(each.value.tags, var.fargate_profile_defaults.tags, {})) } @@ -393,6 +395,8 @@ module "eks_managed_node_group" { # To better understand why this `lookup()` logic is required, see: # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.eks_managed_node_group_defaults, "iam_role_additional_policies", {})) + create_iam_role_policy = try(each.value.create_iam_role_policy, var.eks_managed_node_group_defaults.create_iam_role_policy, true) + iam_role_policy_statements = try(each.value.iam_role_policy_statements, var.eks_managed_node_group_defaults.iam_role_policy_statements, []) # Autoscaling group schedule create_schedule = try(each.value.create_schedule, var.eks_managed_node_group_defaults.create_schedule, true) @@ -534,6 +538,8 @@ module "self_managed_node_group" { # To better understand why this `lookup()` logic is required, see: # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.self_managed_node_group_defaults, "iam_role_additional_policies", {})) + create_iam_role_policy = try(each.value.create_iam_role_policy, var.self_managed_node_group_defaults.create_iam_role_policy, true) + iam_role_policy_statements = try(each.value.iam_role_policy_statements, var.self_managed_node_group_defaults.iam_role_policy_statements, []) # Access entry create_access_entry = try(each.value.create_access_entry, var.self_managed_node_group_defaults.create_access_entry, true) diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index c5bb833d14..654f57b7a8 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -265,6 +265,17 @@ module "eks" { AmazonEC2ContainerRegistryReadOnly = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" additional = aws_iam_policy.node_additional.arn } + iam_role_policy_statements = [ + { + sid = "ECRPullThroughCache" + effect = "Allow" + actions = [ + "ecr:CreateRepository", + "ecr:BatchImportUpstreamImage", + ] + resources = ["*"] + } + ] launch_template_tags = { # enable discovery of autoscaling groups by cluster-autoscaler From c52bbb32a41e5f5cd2b1fd1d9b04ee1baa8e496e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 2 Jul 2024 14:56:53 +0000 Subject: [PATCH 795/962] chore(release): version 20.16.0 [skip ci] ## [20.16.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.15.0...v20.16.0) (2024-07-02) ### Features * Add support for custom IAM role policy ([#3087](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3087)) ([1604c6c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/1604c6cdc8cedcd47b7357c5068dc11d0ed1d7e5)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a98b95b368..918820d974 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.16.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.15.0...v20.16.0) (2024-07-02) + + +### Features + +* Add support for custom IAM role policy ([#3087](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3087)) ([1604c6c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/1604c6cdc8cedcd47b7357c5068dc11d0ed1d7e5)) + ## [20.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.14.0...v20.15.0) (2024-06-27) From ae3379e92429ed842f1c1017fd6ee59ec9f297d4 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 5 Jul 2024 11:42:31 -0400 Subject: [PATCH 796/962] feat: Add support for ML capacity block reservations with EKS managed node group(s) (#3091) --- README.md | 4 ++-- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 6 +++--- examples/karpenter/versions.tf | 2 +- examples/outposts/README.md | 4 ++-- examples/outposts/prerequisites/versions.tf | 2 +- examples/outposts/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/main.tf | 14 ++++++++------ modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/versions.tf | 2 +- tests/fargate-profile/README.md | 4 ++-- tests/fargate-profile/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- versions.tf | 2 +- 24 files changed, 42 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 901dff3675..fa9bcd382d 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.40 | +| [aws](#requirement\_aws) | >= 5.57 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -184,7 +184,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.40 | +| [aws](#provider\_aws) | >= 5.57 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index 6f83215f50..1ad4db0909 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index bc09df1fa5..4f94a5b44f 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -89,7 +89,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.40 | +| [aws](#requirement\_aws) | >= 5.57 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -97,8 +97,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.40 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.40 | +| [aws](#provider\_aws) | >= 5.57 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.57 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index a43b64f4c4..5cd234e754 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } helm = { source = "hashicorp/helm" diff --git a/examples/outposts/README.md b/examples/outposts/README.md index 3779487fde..b2769e0839 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -46,14 +46,14 @@ terraform destroy --auto-approve | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.40 | +| [aws](#requirement\_aws) | >= 5.57 | | [kubernetes](#requirement\_kubernetes) | >= 2.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.40 | +| [aws](#provider\_aws) | >= 5.57 | | [kubernetes](#provider\_kubernetes) | >= 2.20 | ## Modules diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf index 6f83215f50..1ad4db0909 100644 --- a/examples/outposts/prerequisites/versions.tf +++ b/examples/outposts/prerequisites/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } } } diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf index 2ac7910678..3d08b99032 100644 --- a/examples/outposts/versions.tf +++ b/examples/outposts/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index 6f83215f50..1ad4db0909 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } } } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 05f1a0bea2..7109d6f23e 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.40 | +| [aws](#requirement\_aws) | >= 5.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.40 | +| [aws](#provider\_aws) | >= 5.57 | ## Modules diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index facd05a810..50554ae98b 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -203,10 +203,11 @@ resource "aws_launch_template" "this" { } } - # # Set on node group instead - # instance_type = var.launch_template_instance_type - kernel_id = var.kernel_id - key_name = var.key_name + # Instance type(s) are generally set on the node group, + # except when a ML capacity block reseravtion is used + instance_type = var.capacity_type == "CAPACITY_BLOCK" ? element(var.instance_types, 0) : null + kernel_id = var.kernel_id + key_name = var.key_name dynamic "license_specification" { for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} @@ -408,8 +409,9 @@ resource "aws_eks_node_group" "this" { capacity_type = var.capacity_type disk_size = var.use_custom_launch_template ? null : var.disk_size # if using a custom LT, set disk size on custom LT or else it will error here force_update_version = var.force_update_version - instance_types = var.instance_types - labels = var.labels + # ML capacity block reservation requires instance type to be set on the launch template + instance_types = var.capacity_type == "CAPACITY_BLOCK" ? null : var.instance_types + labels = var.labels dynamic "launch_template" { for_each = var.use_custom_launch_template ? [1] : [] diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 6f83215f50..1ad4db0909 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 2396f353ab..609614b9a1 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.40 | +| [aws](#requirement\_aws) | >= 5.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.40 | +| [aws](#provider\_aws) | >= 5.57 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 6f83215f50..1ad4db0909 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index ca1e7bdc0f..eeecd7ba26 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.40 | +| [aws](#requirement\_aws) | >= 5.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.40 | +| [aws](#provider\_aws) | >= 5.57 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 6f83215f50..1ad4db0909 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 05834cc935..ffa6d24837 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.40 | +| [aws](#requirement\_aws) | >= 5.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.40 | +| [aws](#provider\_aws) | >= 5.57 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 6f83215f50..1ad4db0909 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } } } diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 169b4c4f18..9c736bc39b 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.40 | +| [aws](#requirement\_aws) | >= 5.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.40 | +| [aws](#provider\_aws) | >= 5.57 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index 6f83215f50..1ad4db0909 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } } } diff --git a/tests/fargate-profile/README.md b/tests/fargate-profile/README.md index bdb1a909a5..a6980a2c3b 100644 --- a/tests/fargate-profile/README.md +++ b/tests/fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.40 | +| [aws](#requirement\_aws) | >= 5.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.40 | +| [aws](#provider\_aws) | >= 5.57 | ## Modules diff --git a/tests/fargate-profile/versions.tf b/tests/fargate-profile/versions.tf index 6f83215f50..1ad4db0909 100644 --- a/tests/fargate-profile/versions.tf +++ b/tests/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index feebef9c92..0a1f76c2e6 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.40 | +| [aws](#requirement\_aws) | >= 5.57 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.40 | +| [aws](#provider\_aws) | >= 5.57 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index 6f83215f50..1ad4db0909 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } } } diff --git a/versions.tf b/versions.tf index d0f347a88a..6243ed0255 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.40" + version = ">= 5.57" } tls = { source = "hashicorp/tls" From 76d6a64e40e4e0001d70fc715d98234ece32c515 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 5 Jul 2024 15:43:02 +0000 Subject: [PATCH 797/962] chore(release): version 20.17.0 [skip ci] ## [20.17.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.16.0...v20.17.0) (2024-07-05) ### Features * Add support for ML capacity block reservations with EKS managed node group(s) ([#3091](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3091)) ([ae3379e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ae3379e92429ed842f1c1017fd6ee59ec9f297d4)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 918820d974..7baeb7e74c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.17.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.16.0...v20.17.0) (2024-07-05) + + +### Features + +* Add support for ML capacity block reservations with EKS managed node group(s) ([#3091](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3091)) ([ae3379e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ae3379e92429ed842f1c1017fd6ee59ec9f297d4)) + ## [20.16.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.15.0...v20.16.0) (2024-07-02) From f02df92b66a9776a689a2baf39e7474f3b703d89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kir=C3=A1ly=20=C3=81d=C3=A1m?= Date: Fri, 5 Jul 2024 19:12:21 +0200 Subject: [PATCH 798/962] fix: Invoke `aws_iam_session_context` data source only when required (#3058) * fix: Call `aws_iam_session_context` data resource only if needed * Typo. * Move index. * Fix condition. --------- Co-authored-by: Bryant Biggs --- main.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 4cb1200327..ef8b1f6095 100644 --- a/main.tf +++ b/main.tf @@ -2,6 +2,7 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} data "aws_iam_session_context" "current" { + count = (var.create && var.enable_cluster_creator_admin_permissions) || (var.create && var.create_kms_key && local.enable_cluster_encryption_config) ? 1 : 0 # This data source provides information on the IAM source role of an STS assumed role # For non-role ARNs, this data source simply passes the ARN through issuer ARN # Ref https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2327#issuecomment-1355581682 @@ -147,7 +148,7 @@ locals { # better controlled by users through Terraform bootstrap_cluster_creator_admin_permissions = { cluster_creator = { - principal_arn = data.aws_iam_session_context.current.issuer_arn + principal_arn = data.aws_iam_session_context.current[0].issuer_arn type = "STANDARD" policy_associations = { @@ -236,7 +237,7 @@ module "kms" { # Policy enable_default_policy = var.kms_key_enable_default_policy key_owners = var.kms_key_owners - key_administrators = coalescelist(var.kms_key_administrators, [data.aws_iam_session_context.current.issuer_arn]) + key_administrators = coalescelist(var.kms_key_administrators, [data.aws_iam_session_context.current[0].issuer_arn]) key_users = concat([local.cluster_role], var.kms_key_users) key_service_users = var.kms_key_service_users source_policy_documents = var.kms_key_source_policy_documents From 27d649ad04654eb40121088744e9ae87473c6d3a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 5 Jul 2024 17:12:50 +0000 Subject: [PATCH 799/962] chore(release): version 20.17.1 [skip ci] ## [20.17.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.17.0...v20.17.1) (2024-07-05) ### Bug Fixes * Invoke `aws_iam_session_context` data source only when required ([#3058](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3058)) ([f02df92](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f02df92b66a9776a689a2baf39e7474f3b703d89)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7baeb7e74c..425d0519a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.17.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.17.0...v20.17.1) (2024-07-05) + + +### Bug Fixes + +* Invoke `aws_iam_session_context` data source only when required ([#3058](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3058)) ([f02df92](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f02df92b66a9776a689a2baf39e7474f3b703d89)) + ## [20.17.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.16.0...v20.17.0) (2024-07-05) From 93ffdfc6fa380cb0b73df7380e7e62302ebb1a98 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 5 Jul 2024 14:36:49 -0400 Subject: [PATCH 800/962] fix: Revert #3058 - fix: Invoke aws_iam_session_context data source only when required (#3092) fix: Revert #3058 --- main.tf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index ef8b1f6095..4cb1200327 100644 --- a/main.tf +++ b/main.tf @@ -2,7 +2,6 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} data "aws_iam_session_context" "current" { - count = (var.create && var.enable_cluster_creator_admin_permissions) || (var.create && var.create_kms_key && local.enable_cluster_encryption_config) ? 1 : 0 # This data source provides information on the IAM source role of an STS assumed role # For non-role ARNs, this data source simply passes the ARN through issuer ARN # Ref https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2327#issuecomment-1355581682 @@ -148,7 +147,7 @@ locals { # better controlled by users through Terraform bootstrap_cluster_creator_admin_permissions = { cluster_creator = { - principal_arn = data.aws_iam_session_context.current[0].issuer_arn + principal_arn = data.aws_iam_session_context.current.issuer_arn type = "STANDARD" policy_associations = { @@ -237,7 +236,7 @@ module "kms" { # Policy enable_default_policy = var.kms_key_enable_default_policy key_owners = var.kms_key_owners - key_administrators = coalescelist(var.kms_key_administrators, [data.aws_iam_session_context.current[0].issuer_arn]) + key_administrators = coalescelist(var.kms_key_administrators, [data.aws_iam_session_context.current.issuer_arn]) key_users = concat([local.cluster_role], var.kms_key_users) key_service_users = var.kms_key_service_users source_policy_documents = var.kms_key_source_policy_documents From d7aea4ca6b7d5fa08c70bf67fa5e9e5c514d26d2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 5 Jul 2024 18:37:18 +0000 Subject: [PATCH 801/962] chore(release): version 20.17.2 [skip ci] ## [20.17.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.17.1...v20.17.2) (2024-07-05) ### Bug Fixes * Revert [#3058](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3058) - fix: Invoke aws_iam_session_context data source only when required ([#3092](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3092)) ([93ffdfc](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/93ffdfc6fa380cb0b73df7380e7e62302ebb1a98)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 425d0519a0..b631bde8c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.17.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.17.1...v20.17.2) (2024-07-05) + + +### Bug Fixes + +* Revert [#3058](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3058) - fix: Invoke aws_iam_session_context data source only when required ([#3092](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3092)) ([93ffdfc](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/93ffdfc6fa380cb0b73df7380e7e62302ebb1a98)) + ## [20.17.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.17.0...v20.17.1) (2024-07-05) From af88e7d2f835b3dfde242157ba3dd98b749bbc0b Mon Sep 17 00:00:00 2001 From: Taehyun Kim <11684628+kimxogus@users.noreply.github.com> Date: Tue, 16 Jul 2024 08:55:09 +0900 Subject: [PATCH 802/962] feat: Support `bootstrap_self_managed_addons` (#3099) Co-authored-by: Bryant Biggs --- README.md | 5 +++-- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 6 +++--- examples/karpenter/versions.tf | 2 +- examples/outposts/README.md | 4 ++-- examples/outposts/prerequisites/versions.tf | 2 +- examples/outposts/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- main.tf | 9 +++++---- modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/versions.tf | 2 +- tests/fargate-profile/README.md | 4 ++-- tests/fargate-profile/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- variables.tf | 6 ++++++ versions.tf | 2 +- 25 files changed, 46 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index fa9bcd382d..018a39fb5d 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.57 | +| [aws](#requirement\_aws) | >= 5.58 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -184,7 +184,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.57 | +| [aws](#provider\_aws) | >= 5.58 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | @@ -236,6 +236,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [access\_entries](#input\_access\_entries) | Map of access entries to add to the cluster | `any` | `{}` | no | | [attach\_cluster\_encryption\_policy](#input\_attach\_cluster\_encryption\_policy) | Indicates whether or not to attach an additional policy for the cluster IAM role to utilize the encryption key provided | `bool` | `true` | no | | [authentication\_mode](#input\_authentication\_mode) | The authentication mode for the cluster. Valid values are `CONFIG_MAP`, `API` or `API_AND_CONFIG_MAP` | `string` | `"API_AND_CONFIG_MAP"` | no | +| [bootstrap\_self\_managed\_addons](#input\_bootstrap\_self\_managed\_addons) | Indicates whether or not to bootstrap self-managed addons after the cluster has been created | `bool` | `null` | no | | [cloudwatch\_log\_group\_class](#input\_cloudwatch\_log\_group\_class) | Specified the log class of the log group. Possible values are: `STANDARD` or `INFREQUENT_ACCESS` | `string` | `null` | no | | [cloudwatch\_log\_group\_kms\_key\_id](#input\_cloudwatch\_log\_group\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `null` | no | | [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days | `number` | `90` | no | diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index 1ad4db0909..ad809927fc 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 4f94a5b44f..841f512017 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -89,7 +89,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.57 | +| [aws](#requirement\_aws) | >= 5.58 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -97,8 +97,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.57 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.57 | +| [aws](#provider\_aws) | >= 5.58 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.58 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 5cd234e754..1c82fb5584 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } helm = { source = "hashicorp/helm" diff --git a/examples/outposts/README.md b/examples/outposts/README.md index b2769e0839..5f2b1f4038 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -46,14 +46,14 @@ terraform destroy --auto-approve | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.57 | +| [aws](#requirement\_aws) | >= 5.58 | | [kubernetes](#requirement\_kubernetes) | >= 2.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.57 | +| [aws](#provider\_aws) | >= 5.58 | | [kubernetes](#provider\_kubernetes) | >= 2.20 | ## Modules diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf index 1ad4db0909..ad809927fc 100644 --- a/examples/outposts/prerequisites/versions.tf +++ b/examples/outposts/prerequisites/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } } } diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf index 3d08b99032..4a5137beff 100644 --- a/examples/outposts/versions.tf +++ b/examples/outposts/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index 1ad4db0909..ad809927fc 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } } } diff --git a/main.tf b/main.tf index 4cb1200327..04cc950b98 100644 --- a/main.tf +++ b/main.tf @@ -27,10 +27,11 @@ locals { resource "aws_eks_cluster" "this" { count = local.create ? 1 : 0 - name = var.cluster_name - role_arn = local.cluster_role - version = var.cluster_version - enabled_cluster_log_types = var.cluster_enabled_log_types + name = var.cluster_name + role_arn = local.cluster_role + version = var.cluster_version + enabled_cluster_log_types = var.cluster_enabled_log_types + bootstrap_self_managed_addons = var.bootstrap_self_managed_addons access_config { authentication_mode = var.authentication_mode diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 7109d6f23e..22def67a0e 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.57 | +| [aws](#requirement\_aws) | >= 5.58 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.57 | +| [aws](#provider\_aws) | >= 5.58 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 1ad4db0909..ad809927fc 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 609614b9a1..3aa71e8027 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.57 | +| [aws](#requirement\_aws) | >= 5.58 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.57 | +| [aws](#provider\_aws) | >= 5.58 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 1ad4db0909..ad809927fc 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index eeecd7ba26..4528dd40c5 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.57 | +| [aws](#requirement\_aws) | >= 5.58 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.57 | +| [aws](#provider\_aws) | >= 5.58 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 1ad4db0909..ad809927fc 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index ffa6d24837..25b045f691 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.57 | +| [aws](#requirement\_aws) | >= 5.58 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.57 | +| [aws](#provider\_aws) | >= 5.58 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 1ad4db0909..ad809927fc 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } } } diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 9c736bc39b..f9ffad745d 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.57 | +| [aws](#requirement\_aws) | >= 5.58 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.57 | +| [aws](#provider\_aws) | >= 5.58 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index 1ad4db0909..ad809927fc 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } } } diff --git a/tests/fargate-profile/README.md b/tests/fargate-profile/README.md index a6980a2c3b..34728d2236 100644 --- a/tests/fargate-profile/README.md +++ b/tests/fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.57 | +| [aws](#requirement\_aws) | >= 5.58 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.57 | +| [aws](#provider\_aws) | >= 5.58 | ## Modules diff --git a/tests/fargate-profile/versions.tf b/tests/fargate-profile/versions.tf index 1ad4db0909..ad809927fc 100644 --- a/tests/fargate-profile/versions.tf +++ b/tests/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 0a1f76c2e6..52b999ee83 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.57 | +| [aws](#requirement\_aws) | >= 5.58 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.57 | +| [aws](#provider\_aws) | >= 5.58 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index 1ad4db0909..ad809927fc 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } } } diff --git a/variables.tf b/variables.tf index 2d75a4b130..8db1db1c9a 100644 --- a/variables.tf +++ b/variables.tf @@ -136,6 +136,12 @@ variable "cluster_timeouts" { default = {} } +variable "bootstrap_self_managed_addons" { + description = "Indicates whether or not to bootstrap self-managed addons after the cluster has been created" + type = bool + default = null +} + ################################################################################ # Access Entry ################################################################################ diff --git a/versions.tf b/versions.tf index 6243ed0255..a9b4a234c6 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.57" + version = ">= 5.58" } tls = { source = "hashicorp/tls" From f56004dea84ad59dfc846392cd6bd38aa2ee4cab Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 15 Jul 2024 23:55:38 +0000 Subject: [PATCH 803/962] chore(release): version 20.18.0 [skip ci] ## [20.18.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.17.2...v20.18.0) (2024-07-15) ### Features * Support `bootstrap_self_managed_addons` ([#3099](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3099)) ([af88e7d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/af88e7d2f835b3dfde242157ba3dd98b749bbc0b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b631bde8c6..7538c29522 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.18.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.17.2...v20.18.0) (2024-07-15) + + +### Features + +* Support `bootstrap_self_managed_addons` ([#3099](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3099)) ([af88e7d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/af88e7d2f835b3dfde242157ba3dd98b749bbc0b)) + ## [20.17.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.17.1...v20.17.2) (2024-07-05) From e1bb8b66617299c6d9972139b1f9355322e7801e Mon Sep 17 00:00:00 2001 From: Anton Chernev Date: Tue, 16 Jul 2024 00:58:23 +0100 Subject: [PATCH 804/962] feat: Pass the `primary_ipv6` argument to the AWS provider. (#3098) Co-authored-by: Bryant Biggs --- modules/eks-managed-node-group/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 50554ae98b..adaf59cd33 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -268,6 +268,7 @@ resource "aws_launch_template" "this" { ipv6_prefixes = try(network_interfaces.value.ipv6_prefixes, []) network_card_index = try(network_interfaces.value.network_card_index, null) network_interface_id = try(network_interfaces.value.network_interface_id, null) + primary_ipv6 = try(network_interfaces.value.primary_ipv6, null) private_ip_address = try(network_interfaces.value.private_ip_address, null) # Ref: https://github.com/hashicorp/terraform-provider-aws/issues/4570 security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) From e32c29f3b77a3a4c8a7fd3652762e587ae6d1879 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 15 Jul 2024 23:58:53 +0000 Subject: [PATCH 805/962] chore(release): version 20.19.0 [skip ci] ## [20.19.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.18.0...v20.19.0) (2024-07-15) ### Features * Pass the `primary_ipv6` argument to the AWS provider. ([#3098](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3098)) ([e1bb8b6](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e1bb8b66617299c6d9972139b1f9355322e7801e)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7538c29522..a62adbd1fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.19.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.18.0...v20.19.0) (2024-07-15) + + +### Features + +* Pass the `primary_ipv6` argument to the AWS provider. ([#3098](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3098)) ([e1bb8b6](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e1bb8b66617299c6d9972139b1f9355322e7801e)) + ## [20.18.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.17.2...v20.18.0) (2024-07-15) From 1fed91afd6f9372e397002a441a26efd75313e96 Mon Sep 17 00:00:00 2001 From: Koyomi Araragi Date: Tue, 16 Jul 2024 18:10:55 -0300 Subject: [PATCH 806/962] docs: Improve karpenter discovery tags example (#3097) * docs: Improve karpenter discovery tags example * chore: Re-add general tags declaration --------- Co-authored-by: Bryant Biggs --- examples/karpenter/main.tf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 9dd361792b..14435b6354 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -101,12 +101,20 @@ module "eks" { } } - tags = merge(local.tags, { + # cluster_tags = merge(local.tags, { + # NOTE - only use this option if you are using "attach_cluster_primary_security_group" + # and you know what you're doing. In this case, you can remove the "node_security_group_tags" below. + # "karpenter.sh/discovery" = local.name + # }) + + node_security_group_tags = merge(local.tags, { # NOTE - if creating multiple security groups with this module, only tag the # security group that Karpenter should utilize with the following tag # (i.e. - at most, only one security group should have this tag in your account) "karpenter.sh/discovery" = local.name }) + + tags = local.tags } ################################################################################ From 532226e64e61328b25426cabc27e4009e085154f Mon Sep 17 00:00:00 2001 From: Ivan Ka <5395690+ivankatliarchuk@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:19:00 +0100 Subject: [PATCH 807/962] feat: Enable support for ignore_failed_scaling_activities (#3104) * feat(issue-3102): added support for ignore_failed_scaling_activities Signed-off-by: ivan katliarchuk * Update node_groups.tf Co-authored-by: Bryant Biggs * Update modules/self-managed-node-group/variables.tf Co-authored-by: Bryant Biggs * feat(issue-3102): pre-commit checks sorted Signed-off-by: ivan katliarchuk --------- Signed-off-by: ivan katliarchuk Co-authored-by: Bryant Biggs --- modules/self-managed-node-group/README.md | 1 + modules/self-managed-node-group/main.tf | 2 ++ modules/self-managed-node-group/variables.tf | 6 ++++++ node_groups.tf | 2 ++ 4 files changed, 11 insertions(+) diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 25b045f691..0f45ddf244 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -140,6 +140,7 @@ module "self_managed_node_group" { | [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [ignore\_failed\_scaling\_activities](#input\_ignore\_failed\_scaling\_activities) | Whether to ignore failed Auto Scaling scaling activities while waiting for capacity. | `bool` | `null` | no | | [initial\_lifecycle\_hooks](#input\_initial\_lifecycle\_hooks) | One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource | `list(map(string))` | `[]` | no | | [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | | [instance\_maintenance\_policy](#input\_instance\_maintenance\_policy) | If this block is configured, add a instance maintenance policy to the specified Auto Scaling group | `any` | `{}` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index ea1bd8c4ce..02d3384c64 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -565,6 +565,8 @@ resource "aws_autoscaling_group" "this" { min_elb_capacity = var.min_elb_capacity min_size = var.min_size + ignore_failed_scaling_activities = var.ignore_failed_scaling_activities + dynamic "mixed_instances_policy" { for_each = var.use_mixed_instances_policy ? [var.mixed_instances_policy] : [] diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index a3a207050f..f3424784a3 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -408,6 +408,12 @@ variable "desired_size" { default = 1 } +variable "ignore_failed_scaling_activities" { + description = "Whether to ignore failed Auto Scaling scaling activities while waiting for capacity." + type = bool + default = null +} + variable "context" { description = "Reserved" type = string diff --git a/node_groups.tf b/node_groups.tf index 2118c6aaab..e3615bd3ee 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -448,6 +448,8 @@ module "self_managed_node_group" { health_check_type = try(each.value.health_check_type, var.self_managed_node_group_defaults.health_check_type, null) health_check_grace_period = try(each.value.health_check_grace_period, var.self_managed_node_group_defaults.health_check_grace_period, null) + ignore_failed_scaling_activities = try(each.value.ignore_failed_scaling_activities, var.self_managed_node_group_defaults.ignore_failed_scaling_activities, null) + force_delete = try(each.value.force_delete, var.self_managed_node_group_defaults.force_delete, null) force_delete_warm_pool = try(each.value.force_delete_warm_pool, var.self_managed_node_group_defaults.force_delete_warm_pool, null) termination_policies = try(each.value.termination_policies, var.self_managed_node_group_defaults.termination_policies, []) From 705eb80cfacf563f74f760261c2154d2fd9f1d87 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 19 Jul 2024 14:19:40 +0000 Subject: [PATCH 808/962] chore(release): version 20.20.0 [skip ci] ## [20.20.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.19.0...v20.20.0) (2024-07-19) ### Features * Enable support for ignore_failed_scaling_activities ([#3104](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3104)) ([532226e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/532226e64e61328b25426cabc27e4009e085154f)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a62adbd1fc..038167934e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.20.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.19.0...v20.20.0) (2024-07-19) + + +### Features + +* Enable support for ignore_failed_scaling_activities ([#3104](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3104)) ([532226e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/532226e64e61328b25426cabc27e4009e085154f)) + ## [20.19.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.18.0...v20.19.0) (2024-07-15) From e12ab7a5de4ac82968aaede419752ce2bbb6a93d Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 5 Aug 2024 09:07:13 -0500 Subject: [PATCH 809/962] feat: Add support for `upgrade_policy` (#3112) * feat: Add support for `upgrade_policy` * chore: Update test and validate --- .pre-commit-config.yaml | 2 +- README.md | 5 +++-- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 6 +++--- examples/karpenter/versions.tf | 2 +- examples/outposts/README.md | 4 ++-- examples/outposts/prerequisites/versions.tf | 2 +- examples/outposts/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- main.tf | 8 ++++++++ modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/main.tf | 6 +++++- tests/eks-managed-node-group/versions.tf | 2 +- tests/fargate-profile/README.md | 4 ++-- tests/fargate-profile/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- variables.tf | 6 ++++++ versions.tf | 2 +- 27 files changed, 55 insertions(+), 36 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 00b113536e..fc21bc9c25 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.92.0 + rev: v1.92.1 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index 018a39fb5d..6826371bae 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.58 | +| [aws](#requirement\_aws) | >= 5.61 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -184,7 +184,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.58 | +| [aws](#provider\_aws) | >= 5.61 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | @@ -267,6 +267,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_service\_ipv6\_cidr](#input\_cluster\_service\_ipv6\_cidr) | The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | +| [cluster\_upgrade\_policy](#input\_cluster\_upgrade\_policy) | Configuration block for the cluster upgrade policy | `any` | `{}` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.27`) | `string` | `null` | no | | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index ad809927fc..5bfe6da389 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 841f512017..e6b8a1e333 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -89,7 +89,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.58 | +| [aws](#requirement\_aws) | >= 5.61 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -97,8 +97,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.58 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.58 | +| [aws](#provider\_aws) | >= 5.61 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.61 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 1c82fb5584..8a0624ba0d 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } helm = { source = "hashicorp/helm" diff --git a/examples/outposts/README.md b/examples/outposts/README.md index 5f2b1f4038..1dcab73cca 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -46,14 +46,14 @@ terraform destroy --auto-approve | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.58 | +| [aws](#requirement\_aws) | >= 5.61 | | [kubernetes](#requirement\_kubernetes) | >= 2.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.58 | +| [aws](#provider\_aws) | >= 5.61 | | [kubernetes](#provider\_kubernetes) | >= 2.20 | ## Modules diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf index ad809927fc..5bfe6da389 100644 --- a/examples/outposts/prerequisites/versions.tf +++ b/examples/outposts/prerequisites/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } } } diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf index 4a5137beff..9836b3a468 100644 --- a/examples/outposts/versions.tf +++ b/examples/outposts/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index ad809927fc..5bfe6da389 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } } } diff --git a/main.tf b/main.tf index 04cc950b98..47bb55a2cf 100644 --- a/main.tf +++ b/main.tf @@ -84,6 +84,14 @@ resource "aws_eks_cluster" "this" { } } + dynamic "upgrade_policy" { + for_each = length(var.cluster_upgrade_policy) > 0 ? [var.cluster_upgrade_policy] : [] + + content { + support_type = try(upgrade_policy.value.support_type, null) + } + } + tags = merge( { terraform-aws-modules = "eks" }, var.tags, diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 22def67a0e..46e36c06ac 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.58 | +| [aws](#requirement\_aws) | >= 5.61 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.58 | +| [aws](#provider\_aws) | >= 5.61 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index ad809927fc..5bfe6da389 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 3aa71e8027..b1ee9e0ade 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.58 | +| [aws](#requirement\_aws) | >= 5.61 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.58 | +| [aws](#provider\_aws) | >= 5.61 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index ad809927fc..5bfe6da389 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 4528dd40c5..1bcf514f47 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.58 | +| [aws](#requirement\_aws) | >= 5.61 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.58 | +| [aws](#provider\_aws) | >= 5.61 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index ad809927fc..5bfe6da389 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 0f45ddf244..48662aabad 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.58 | +| [aws](#requirement\_aws) | >= 5.61 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.58 | +| [aws](#provider\_aws) | >= 5.61 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index ad809927fc..5bfe6da389 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } } } diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index f9ffad745d..8a0bfb485a 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.58 | +| [aws](#requirement\_aws) | >= 5.61 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.58 | +| [aws](#provider\_aws) | >= 5.61 | ## Modules diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index 654f57b7a8..2cca15db06 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -7,7 +7,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.29" + cluster_version = "1.30" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -61,6 +61,10 @@ module "eks" { } } + cluster_upgrade_policy = { + support_type = "STANDARD" + } + vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index ad809927fc..5bfe6da389 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } } } diff --git a/tests/fargate-profile/README.md b/tests/fargate-profile/README.md index 34728d2236..cb0fa88208 100644 --- a/tests/fargate-profile/README.md +++ b/tests/fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.58 | +| [aws](#requirement\_aws) | >= 5.61 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.58 | +| [aws](#provider\_aws) | >= 5.61 | ## Modules diff --git a/tests/fargate-profile/versions.tf b/tests/fargate-profile/versions.tf index ad809927fc..5bfe6da389 100644 --- a/tests/fargate-profile/versions.tf +++ b/tests/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 52b999ee83..c6eb533114 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.58 | +| [aws](#requirement\_aws) | >= 5.61 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.58 | +| [aws](#provider\_aws) | >= 5.61 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index ad809927fc..5bfe6da389 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } } } diff --git a/variables.tf b/variables.tf index 8db1db1c9a..420248c744 100644 --- a/variables.tf +++ b/variables.tf @@ -44,6 +44,12 @@ variable "authentication_mode" { default = "API_AND_CONFIG_MAP" } +variable "cluster_upgrade_policy" { + description = "Configuration block for the cluster upgrade policy" + type = any + default = {} +} + variable "cluster_additional_security_group_ids" { description = "List of additional, externally created security group IDs to attach to the cluster control plane" type = list(string) diff --git a/versions.tf b/versions.tf index a9b4a234c6..00d8754704 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.58" + version = ">= 5.61" } tls = { source = "hashicorp/tls" From bfa5821113a2304f2637cc1f6aef35628d24b2e9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 5 Aug 2024 14:07:46 +0000 Subject: [PATCH 810/962] chore(release): version 20.21.0 [skip ci] ## [20.21.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.20.0...v20.21.0) (2024-08-05) ### Features * Add support for `upgrade_policy` ([#3112](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3112)) ([e12ab7a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e12ab7a5de4ac82968aaede419752ce2bbb6a93d)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 038167934e..d9a3211aa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.21.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.20.0...v20.21.0) (2024-08-05) + + +### Features + +* Add support for `upgrade_policy` ([#3112](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3112)) ([e12ab7a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e12ab7a5de4ac82968aaede419752ce2bbb6a93d)) + ## [20.20.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.19.0...v20.20.0) (2024-07-19) From 75db486530459a04ce6eb2e4ed44b29d062de1b3 Mon Sep 17 00:00:00 2001 From: Josephuss <32011432+Josephuss@users.noreply.github.com> Date: Mon, 5 Aug 2024 23:13:39 +0800 Subject: [PATCH 811/962] feat: Enable update in place for node groups with cluster placement group strategy (#3045) * feat(eks):added subnet az filter for eks nodegroup placement groups * fix: Correct logice for restricting placement group to AZ * fix: Ensure node group args are passed from root module --------- Co-authored-by: Bryant Biggs --- modules/eks-managed-node-group/README.md | 3 +- modules/eks-managed-node-group/main.tf | 43 +++++++++++++++----- modules/eks-managed-node-group/variables.tf | 7 ++++ modules/self-managed-node-group/README.md | 4 +- modules/self-managed-node-group/main.tf | 35 ++++++++++++---- modules/self-managed-node-group/variables.tf | 12 ++++++ node_groups.tf | 5 ++- tests/eks-managed-node-group/main.tf | 13 +++--- 8 files changed, 97 insertions(+), 25 deletions(-) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 46e36c06ac..96b6c4f290 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -97,7 +97,7 @@ module "eks_managed_node_group" { | [aws_iam_policy_document.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_ssm_parameter.ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | -| [aws_subnets.efa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | +| [aws_subnets.placement_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs @@ -169,6 +169,7 @@ module "eks_managed_node_group" { | [name](#input\_name) | Name of the EKS managed node group | `string` | `""` | no | | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | | [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | +| [placement\_group\_az](#input\_placement\_group\_az) | Availability zone where placement group is created (ex. `eu-west-1c`) | `string` | `null` | no | | [placement\_group\_strategy](#input\_placement\_group\_strategy) | The placement group strategy | `string` | `"cluster"` | no | | [platform](#input\_platform) | [DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | `string` | `"linux"` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index adaf59cd33..9c0c78c9c4 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -39,6 +39,8 @@ data "aws_ec2_instance_type" "this" { } locals { + enable_efa_support = var.create && var.enable_efa_support + efa_instance_type = try(element(var.instance_types, 0), "") num_network_cards = try(data.aws_ec2_instance_type.this[0].maximum_network_cards, 0) @@ -52,7 +54,7 @@ locals { } ] - network_interfaces = var.enable_efa_support ? local.efa_network_interfaces : var.network_interfaces + network_interfaces = local.enable_efa_support ? local.efa_network_interfaces : var.network_interfaces } ################################################################################ @@ -63,7 +65,7 @@ locals { launch_template_name = coalesce(var.launch_template_name, "${var.name}-eks-node-group") security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) - placement = var.create && (var.enable_efa_support || var.create_placement_group) ? { group_name = aws_placement_group.this[0].name } : var.placement + placement = local.create_placement_group ? { group_name = aws_placement_group.this[0].name } : var.placement } resource "aws_launch_template" "this" { @@ -390,7 +392,7 @@ resource "aws_eks_node_group" "this" { # Required cluster_name = var.cluster_name node_role_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn - subnet_ids = var.enable_efa_support ? data.aws_subnets.efa[0].ids : var.subnet_ids + subnet_ids = local.create_placement_group ? data.aws_subnets.placement_group[0].ids : var.subnet_ids scaling_config { min_size = var.min_size @@ -605,8 +607,12 @@ resource "aws_iam_role_policy" "this" { # Placement Group ################################################################################ +locals { + create_placement_group = var.create && (local.enable_efa_support || var.create_placement_group) +} + resource "aws_placement_group" "this" { - count = var.create && (var.enable_efa_support || var.create_placement_group) ? 1 : 0 + count = local.create_placement_group ? 1 : 0 name = "${var.cluster_name}-${var.name}" strategy = var.placement_group_strategy @@ -624,8 +630,11 @@ resource "aws_placement_group" "this" { ################################################################################ # Find the availability zones supported by the instance type +# TODO - remove at next breaking change +# Force users to be explicit about which AZ to use when using placement groups, +# with or without EFA support data "aws_ec2_instance_type_offerings" "this" { - count = var.create && var.enable_efa_support ? 1 : 0 + count = local.enable_efa_support ? 1 : 0 filter { name = "instance-type" @@ -637,17 +646,31 @@ data "aws_ec2_instance_type_offerings" "this" { # Reverse the lookup to find one of the subnets provided based on the availability # availability zone ID of the queried instance type (supported) -data "aws_subnets" "efa" { - count = var.create && var.enable_efa_support ? 1 : 0 +data "aws_subnets" "placement_group" { + count = local.create_placement_group ? 1 : 0 filter { name = "subnet-id" values = var.subnet_ids } - filter { - name = "availability-zone-id" - values = data.aws_ec2_instance_type_offerings.this[0].locations + # The data source can lookup the first available AZ or you can specify an AZ (next filter) + dynamic "filter" { + for_each = var.enable_efa_support && var.placement_group_az == null ? [1] : [] + + content { + name = "availability-zone-id" + values = data.aws_ec2_instance_type_offerings.this[0].locations + } + } + + dynamic "filter" { + for_each = var.placement_group_az != null ? [var.placement_group_az] : [] + + content { + name = "availability-zone" + values = [filter.value] + } } } diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 503f617789..bb60b85665 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -303,6 +303,7 @@ variable "create_placement_group" { default = false } +# TODO - remove at next breaking change variable "placement_group_strategy" { description = "The placement group strategy" type = string @@ -337,6 +338,12 @@ variable "subnet_ids" { default = null } +variable "placement_group_az" { + description = "Availability zone where placement group is created (ex. `eu-west-1c`)" + type = string + default = null +} + variable "min_size" { description = "Minimum number of instances/nodes" type = number diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 48662aabad..8422b0c7ef 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -78,7 +78,7 @@ module "self_managed_node_group" { | [aws_iam_policy_document.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_ssm_parameter.ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | -| [aws_subnets.efa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | +| [aws_subnets.placement_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs @@ -110,6 +110,7 @@ module "self_managed_node_group" { | [create\_iam\_instance\_profile](#input\_create\_iam\_instance\_profile) | Determines whether an IAM instance profile is created or to use an existing IAM instance profile | `bool` | `true` | no | | [create\_iam\_role\_policy](#input\_create\_iam\_role\_policy) | Determines whether an IAM role policy is created or not | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create launch template or not | `bool` | `true` | no | +| [create\_placement\_group](#input\_create\_placement\_group) | Determines whether a placement group is created & used by the node group | `bool` | `false` | no | | [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | | [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | | [default\_cooldown](#input\_default\_cooldown) | The amount of time, in seconds, after a scaling activity completes before another scaling activity can start | `number` | `null` | no | @@ -170,6 +171,7 @@ module "self_managed_node_group" { | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | | [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | | [placement\_group](#input\_placement\_group) | The name of the placement group into which you'll launch your instances, if any | `string` | `null` | no | +| [placement\_group\_az](#input\_placement\_group\_az) | Availability zone where placement group is created (ex. `eu-west-1c`) | `string` | `null` | no | | [platform](#input\_platform) | [DEPRECATED - must use `ami_type` instead. Will be removed in `v21.0`] | `string` | `null` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | | [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 02d3384c64..42411c484e 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -745,7 +745,7 @@ resource "aws_autoscaling_group" "this" { target_group_arns = var.target_group_arns termination_policies = var.termination_policies - vpc_zone_identifier = local.enable_efa_support ? data.aws_subnets.efa[0].ids : var.subnet_ids + vpc_zone_identifier = local.enable_efa_support ? data.aws_subnets.placement_group[0].ids : var.subnet_ids wait_for_capacity_timeout = var.wait_for_capacity_timeout wait_for_elb_capacity = var.wait_for_elb_capacity @@ -930,8 +930,12 @@ resource "aws_iam_role_policy" "this" { # Placement Group ################################################################################ +locals { + create_placement_group = var.create && (local.enable_efa_support || var.create_placement_group) +} + resource "aws_placement_group" "this" { - count = local.enable_efa_support ? 1 : 0 + count = local.create_placement_group ? 1 : 0 name = "${var.cluster_name}-${var.name}" strategy = "cluster" @@ -949,6 +953,9 @@ resource "aws_placement_group" "this" { ################################################################################ # Find the availability zones supported by the instance type +# TODO - remove at next breaking change +# Force users to be explicit about which AZ to use when using placement groups, +# with or without EFA support data "aws_ec2_instance_type_offerings" "this" { count = local.enable_efa_support ? 1 : 0 @@ -962,17 +969,31 @@ data "aws_ec2_instance_type_offerings" "this" { # Reverse the lookup to find one of the subnets provided based on the availability # availability zone ID of the queried instance type (supported) -data "aws_subnets" "efa" { - count = local.enable_efa_support ? 1 : 0 +data "aws_subnets" "placement_group" { + count = local.create_placement_group ? 1 : 0 filter { name = "subnet-id" values = var.subnet_ids } - filter { - name = "availability-zone-id" - values = data.aws_ec2_instance_type_offerings.this[0].locations + # The data source can lookup the first available AZ or you can specify an AZ (next filter) + dynamic "filter" { + for_each = local.create_placement_group && var.placement_group_az == null ? [1] : [] + + content { + name = "availability-zone-id" + values = data.aws_ec2_instance_type_offerings.this[0].locations + } + } + + dynamic "filter" { + for_each = var.placement_group_az != null ? [var.placement_group_az] : [] + + content { + name = "availability-zone" + values = [filter.value] + } } } diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index f3424784a3..92121ea750 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -256,6 +256,12 @@ variable "placement" { default = {} } +variable "create_placement_group" { + description = "Determines whether a placement group is created & used by the node group" + type = bool + default = false +} + variable "private_dns_name_options" { description = "The options for the instance hostname. The default values are inherited from the subnet" type = map(string) @@ -384,6 +390,12 @@ variable "availability_zones" { default = null } +variable "placement_group_az" { + description = "Availability zone where placement group is created (ex. `eu-west-1c`)" + type = string + default = null +} + variable "subnet_ids" { description = "A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones`" type = list(string) diff --git a/node_groups.tf b/node_groups.tf index e3615bd3ee..de75fc9133 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -376,9 +376,10 @@ module "eks_managed_node_group" { enable_monitoring = try(each.value.enable_monitoring, var.eks_managed_node_group_defaults.enable_monitoring, true) enable_efa_support = try(each.value.enable_efa_support, var.eks_managed_node_group_defaults.enable_efa_support, false) create_placement_group = try(each.value.create_placement_group, var.eks_managed_node_group_defaults.create_placement_group, false) + placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, {}) + placement_group_az = try(each.value.placement_group_az, var.eks_managed_node_group_defaults.placement_group_az, null) placement_group_strategy = try(each.value.placement_group_strategy, var.eks_managed_node_group_defaults.placement_group_strategy, "cluster") network_interfaces = try(each.value.network_interfaces, var.eks_managed_node_group_defaults.network_interfaces, []) - placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, {}) maintenance_options = try(each.value.maintenance_options, var.eks_managed_node_group_defaults.maintenance_options, {}) private_dns_name_options = try(each.value.private_dns_name_options, var.eks_managed_node_group_defaults.private_dns_name_options, {}) @@ -444,7 +445,9 @@ module "self_managed_node_group" { context = try(each.value.context, var.self_managed_node_group_defaults.context, null) target_group_arns = try(each.value.target_group_arns, var.self_managed_node_group_defaults.target_group_arns, []) + create_placement_group = try(each.value.create_placement_group, var.self_managed_node_group_defaults.create_placement_group, false) placement_group = try(each.value.placement_group, var.self_managed_node_group_defaults.placement_group, null) + placement_group_az = try(each.value.placement_group_az, var.self_managed_node_group_defaults.placement_group_az, null) health_check_type = try(each.value.health_check_type, var.self_managed_node_group_defaults.health_check_type, null) health_check_grace_period = try(each.value.health_check_grace_period, var.self_managed_node_group_defaults.health_check_grace_period, null) diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index 2cca15db06..dfe7aea768 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -90,11 +90,16 @@ module "eks" { } } + placement_group = { + create_placement_group = true + # forces the subnet lookup to be restricted to this availability zone + placement_group_az = element(local.azs, 3) + } + # AL2023 node group utilizing new user data format which utilizes nodeadm # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh) al2023_nodeadm = { - ami_type = "AL2023_x86_64_STANDARD" - + ami_type = "AL2023_x86_64_STANDARD" use_latest_ami_release_version = true cloudinit_pre_nodeadm = [ @@ -376,9 +381,7 @@ module "eks_managed_node_group" { subnet_ids = module.vpc.private_subnets cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id - vpc_security_group_ids = [ - module.eks.node_security_group_id, - ] + vpc_security_group_ids = [module.eks.node_security_group_id] ami_type = "BOTTLEROCKET_x86_64" From 5fe865e860c4cc8506c639f2e63bc25e21a31b37 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 5 Aug 2024 15:14:08 +0000 Subject: [PATCH 812/962] chore(release): version 20.22.0 [skip ci] ## [20.22.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.21.0...v20.22.0) (2024-08-05) ### Features * Enable update in place for node groups with cluster placement group strategy ([#3045](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3045)) ([75db486](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/75db486530459a04ce6eb2e4ed44b29d062de1b3)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9a3211aa4..29a23f8720 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.22.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.21.0...v20.22.0) (2024-08-05) + + +### Features + +* Enable update in place for node groups with cluster placement group strategy ([#3045](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3045)) ([75db486](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/75db486530459a04ce6eb2e4ed44b29d062de1b3)) + ## [20.21.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.20.0...v20.21.0) (2024-08-05) From 6124a08578d6c6bca1851df4c82cb7e2126e460a Mon Sep 17 00:00:00 2001 From: Loren Gordon Date: Fri, 9 Aug 2024 15:23:01 -0700 Subject: [PATCH 813/962] fix: Eliminates null check on tag values to fix for_each error about unknown *keys* (#3119) Eliminates null check on tag values to fix for_each error about unknown *keys* Fixes #3118 Related #2760 Related #2681 Related #2337 --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 47bb55a2cf..1846a1da05 100644 --- a/main.tf +++ b/main.tf @@ -124,7 +124,7 @@ resource "aws_ec2_tag" "cluster_primary_security_group" { # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2006 # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2008 for_each = { for k, v in merge(var.tags, var.cluster_tags) : - k => v if local.create && k != "Name" && var.create_cluster_primary_security_group_tags && v != null + k => v if local.create && k != "Name" && var.create_cluster_primary_security_group_tags } resource_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id From faa42df3a30b163eac5ba6f0fcdf93c54b5b68ef Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 9 Aug 2024 22:23:31 +0000 Subject: [PATCH 814/962] chore(release): version 20.22.1 [skip ci] ## [20.22.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.22.0...v20.22.1) (2024-08-09) ### Bug Fixes * Eliminates null check on tag values to fix for_each error about unknown *keys* ([#3119](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3119)) ([6124a08](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6124a08578d6c6bca1851df4c82cb7e2126e460a)), closes [#3118](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3118) [#2760](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2760) [#2681](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2681) [#2337](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2337) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29a23f8720..2709249ae7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.22.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.22.0...v20.22.1) (2024-08-09) + + +### Bug Fixes + +* Eliminates null check on tag values to fix for_each error about unknown *keys* ([#3119](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3119)) ([6124a08](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6124a08578d6c6bca1851df4c82cb7e2126e460a)), closes [#3118](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3118) [#2760](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2760) [#2681](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2681) [#2337](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2337) + ## [20.22.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.21.0...v20.22.0) (2024-08-05) From 72668ac04a2879fd3294e6059238b4aed57278fa Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 9 Aug 2024 17:39:56 -0500 Subject: [PATCH 815/962] feat: Add new output values for OIDC issuer URL and provider that are dual-stack compatible (#3120) * feat: Add new output values for OIDC issuer URL and provider that support IPv4/IPv6 dualstack * chore: Revert addition of `dualstack_oidc_provider` * fix: Add check for `aws` partition since this is the only partition currently supported * fix: Revert partition conditional logic --- README.md | 1 + examples/karpenter/README.md | 1 + examples/karpenter/outputs.tf | 5 +++++ examples/outposts/README.md | 1 + examples/outposts/outputs.tf | 5 +++++ outputs.tf | 9 +++++++++ tests/eks-managed-node-group/README.md | 1 + tests/eks-managed-node-group/outputs.tf | 5 +++++ tests/fargate-profile/README.md | 1 + tests/fargate-profile/outputs.tf | 5 +++++ tests/self-managed-node-group/README.md | 1 + tests/self-managed-node-group/outputs.tf | 5 +++++ 12 files changed, 40 insertions(+) diff --git a/README.md b/README.md index 6826371bae..6aad481af7 100644 --- a/README.md +++ b/README.md @@ -335,6 +335,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | | [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index e6b8a1e333..62291cdc07 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -136,6 +136,7 @@ No inputs. | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | | [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | diff --git a/examples/karpenter/outputs.tf b/examples/karpenter/outputs.tf index de0e2e6a28..d85a8b034e 100644 --- a/examples/karpenter/outputs.tf +++ b/examples/karpenter/outputs.tf @@ -32,6 +32,11 @@ output "cluster_oidc_issuer_url" { value = module.eks.cluster_oidc_issuer_url } +output "cluster_dualstack_oidc_issuer_url" { + description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_dualstack_oidc_issuer_url +} + output "cluster_platform_version" { description = "Platform version for the cluster" value = module.eks.cluster_platform_version diff --git a/examples/outposts/README.md b/examples/outposts/README.md index 1dcab73cca..5057fc4a5e 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -90,6 +90,7 @@ terraform destroy --auto-approve | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | | [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | diff --git a/examples/outposts/outputs.tf b/examples/outposts/outputs.tf index 24183fd207..9357464c29 100644 --- a/examples/outposts/outputs.tf +++ b/examples/outposts/outputs.tf @@ -32,6 +32,11 @@ output "cluster_oidc_issuer_url" { value = module.eks.cluster_oidc_issuer_url } +output "cluster_dualstack_oidc_issuer_url" { + description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_dualstack_oidc_issuer_url +} + output "cluster_platform_version" { description = "Platform version for the cluster" value = module.eks.cluster_platform_version diff --git a/outputs.tf b/outputs.tf index 45b68a4a23..0bd2d10f4e 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,3 +1,7 @@ +locals { + dualstack_oidc_issuer_url = try(replace(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://oidc.eks.", "https://oidc-eks."), ".amazonaws.com/", ".api.aws/"), null) +} + ################################################################################ # Cluster ################################################################################ @@ -52,6 +56,11 @@ output "cluster_oidc_issuer_url" { value = try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, null) } +output "cluster_dualstack_oidc_issuer_url" { + description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" + value = local.dualstack_oidc_issuer_url +} + output "cluster_version" { description = "The Kubernetes version for the cluster" value = try(aws_eks_cluster.this[0].version, null) diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 8a0bfb485a..f6f753897b 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -65,6 +65,7 @@ No inputs. | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | | [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | diff --git a/tests/eks-managed-node-group/outputs.tf b/tests/eks-managed-node-group/outputs.tf index 24183fd207..9357464c29 100644 --- a/tests/eks-managed-node-group/outputs.tf +++ b/tests/eks-managed-node-group/outputs.tf @@ -32,6 +32,11 @@ output "cluster_oidc_issuer_url" { value = module.eks.cluster_oidc_issuer_url } +output "cluster_dualstack_oidc_issuer_url" { + description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_dualstack_oidc_issuer_url +} + output "cluster_platform_version" { description = "Platform version for the cluster" value = module.eks.cluster_platform_version diff --git a/tests/fargate-profile/README.md b/tests/fargate-profile/README.md index cb0fa88208..f0ac3d5390 100644 --- a/tests/fargate-profile/README.md +++ b/tests/fargate-profile/README.md @@ -56,6 +56,7 @@ No inputs. | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | | [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | diff --git a/tests/fargate-profile/outputs.tf b/tests/fargate-profile/outputs.tf index 24183fd207..9357464c29 100644 --- a/tests/fargate-profile/outputs.tf +++ b/tests/fargate-profile/outputs.tf @@ -32,6 +32,11 @@ output "cluster_oidc_issuer_url" { value = module.eks.cluster_oidc_issuer_url } +output "cluster_dualstack_oidc_issuer_url" { + description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_dualstack_oidc_issuer_url +} + output "cluster_platform_version" { description = "Platform version for the cluster" value = module.eks.cluster_platform_version diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index c6eb533114..6f5ddce892 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -61,6 +61,7 @@ No inputs. | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | | [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | diff --git a/tests/self-managed-node-group/outputs.tf b/tests/self-managed-node-group/outputs.tf index 24183fd207..9357464c29 100644 --- a/tests/self-managed-node-group/outputs.tf +++ b/tests/self-managed-node-group/outputs.tf @@ -32,6 +32,11 @@ output "cluster_oidc_issuer_url" { value = module.eks.cluster_oidc_issuer_url } +output "cluster_dualstack_oidc_issuer_url" { + description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_dualstack_oidc_issuer_url +} + output "cluster_platform_version" { description = "Platform version for the cluster" value = module.eks.cluster_platform_version From 1360e3de68d4b18048477cef43d74c3dc0dd081b Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 9 Aug 2024 22:40:26 +0000 Subject: [PATCH 816/962] chore(release): version 20.23.0 [skip ci] ## [20.23.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.22.1...v20.23.0) (2024-08-09) ### Features * Add new output values for OIDC issuer URL and provider that are dual-stack compatible ([#3120](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3120)) ([72668ac](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/72668ac04a2879fd3294e6059238b4aed57278fa)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2709249ae7..be11e0f6d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.23.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.22.1...v20.23.0) (2024-08-09) + + +### Features + +* Add new output values for OIDC issuer URL and provider that are dual-stack compatible ([#3120](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3120)) ([72668ac](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/72668ac04a2879fd3294e6059238b4aed57278fa)) + ## [20.22.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.22.0...v20.22.1) (2024-08-09) From e31765153570631c1978e11cfd1d28e5fc349d8f Mon Sep 17 00:00:00 2001 From: Andy Townsend Date: Mon, 19 Aug 2024 20:31:36 +0100 Subject: [PATCH 817/962] feat: Add support for Karpenter v1 controller IAM role permissions (#3126) * chore: update controller IAM role permissions to support karpenter v1 * Update versions.tf * Revert "Update versions.tf" This reverts commit f0e5c791443301ef7f802c627efe7f7226b95046. * fix: Add support for both v1 and prior to v1 controller permission policy --------- Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 2 +- examples/karpenter/main.tf | 4 +- modules/karpenter/README.md | 3 + modules/karpenter/main.tf | 341 +-------------- modules/karpenter/policy.tf | 749 +++++++++++++++++++++++++++++++++ modules/karpenter/variables.tf | 7 + 6 files changed, 764 insertions(+), 342 deletions(-) create mode 100644 modules/karpenter/policy.tf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc21bc9c25..b73ed98715 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.92.1 + rev: v1.92.2 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 14435b6354..72ceff275a 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -126,6 +126,8 @@ module "karpenter" { cluster_name = module.eks.cluster_name + enable_v1_permissions = true + enable_pod_identity = true create_pod_identity_association = true @@ -155,7 +157,7 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "0.37.0" + version = "1.0.0" wait = false values = [ diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 1bcf514f47..6810d0fa0c 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -121,6 +121,8 @@ No modules. | [aws_iam_policy_document.controller_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.node_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.v033](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.v1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | @@ -141,6 +143,7 @@ No modules. | [enable\_irsa](#input\_enable\_irsa) | Determines whether to enable support for IAM role for service accounts | `bool` | `false` | no | | [enable\_pod\_identity](#input\_enable\_pod\_identity) | Determines whether to enable support for EKS pod identity | `bool` | `true` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | +| [enable\_v1\_permissions](#input\_enable\_v1\_permissions) | Determines whether to enable permissions suitable for v1+ (`true`) or for v0.33.x-v0.37.x (`false`) | `bool` | `false` | no | | [iam\_policy\_description](#input\_iam\_policy\_description) | IAM policy description | `string` | `"Karpenter controller IAM policy"` | no | | [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy | `string` | `"KarpenterController"` | no | | [iam\_policy\_path](#input\_iam\_policy\_path) | Path of the IAM policy | `string` | `"/"` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index fee75db43a..fc4a5dca57 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -84,346 +84,7 @@ resource "aws_iam_role" "controller" { data "aws_iam_policy_document" "controller" { count = local.create_iam_role ? 1 : 0 - statement { - sid = "AllowScopedEC2InstanceActions" - resources = [ - "arn:${local.partition}:ec2:*::image/*", - "arn:${local.partition}:ec2:*::snapshot/*", - "arn:${local.partition}:ec2:*:*:spot-instances-request/*", - "arn:${local.partition}:ec2:*:*:security-group/*", - "arn:${local.partition}:ec2:*:*:subnet/*", - "arn:${local.partition}:ec2:*:*:launch-template/*", - ] - - actions = [ - "ec2:RunInstances", - "ec2:CreateFleet" - ] - } - - statement { - sid = "AllowScopedEC2InstanceActionsWithTags" - resources = [ - "arn:${local.partition}:ec2:*:*:fleet/*", - "arn:${local.partition}:ec2:*:*:instance/*", - "arn:${local.partition}:ec2:*:*:volume/*", - "arn:${local.partition}:ec2:*:*:network-interface/*", - "arn:${local.partition}:ec2:*:*:launch-template/*", - "arn:${local.partition}:ec2:*:*:spot-instances-request/*", - ] - actions = [ - "ec2:RunInstances", - "ec2:CreateFleet", - "ec2:CreateLaunchTemplate" - ] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringLike" - variable = "aws:RequestTag/karpenter.sh/nodepool" - values = ["*"] - } - } - - statement { - sid = "AllowScopedResourceCreationTagging" - resources = [ - "arn:${local.partition}:ec2:*:*:fleet/*", - "arn:${local.partition}:ec2:*:*:instance/*", - "arn:${local.partition}:ec2:*:*:volume/*", - "arn:${local.partition}:ec2:*:*:network-interface/*", - "arn:${local.partition}:ec2:*:*:launch-template/*", - "arn:${local.partition}:ec2:*:*:spot-instances-request/*", - ] - actions = ["ec2:CreateTags"] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "ec2:CreateAction" - values = [ - "RunInstances", - "CreateFleet", - "CreateLaunchTemplate", - ] - } - - condition { - test = "StringLike" - variable = "aws:RequestTag/karpenter.sh/nodepool" - values = ["*"] - } - } - - statement { - sid = "AllowScopedResourceTagging" - resources = ["arn:${local.partition}:ec2:*:*:instance/*"] - actions = ["ec2:CreateTags"] - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringLike" - variable = "aws:ResourceTag/karpenter.sh/nodepool" - values = ["*"] - } - - condition { - test = "ForAllValues:StringEquals" - variable = "aws:TagKeys" - values = [ - "karpenter.sh/nodeclaim", - "Name", - ] - } - } - - statement { - sid = "AllowScopedDeletion" - resources = [ - "arn:${local.partition}:ec2:*:*:instance/*", - "arn:${local.partition}:ec2:*:*:launch-template/*" - ] - - actions = [ - "ec2:TerminateInstances", - "ec2:DeleteLaunchTemplate" - ] - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringLike" - variable = "aws:ResourceTag/karpenter.sh/nodepool" - values = ["*"] - } - } - - statement { - sid = "AllowRegionalReadActions" - resources = ["*"] - actions = [ - "ec2:DescribeAvailabilityZones", - "ec2:DescribeImages", - "ec2:DescribeInstances", - "ec2:DescribeInstanceTypeOfferings", - "ec2:DescribeInstanceTypes", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSpotPriceHistory", - "ec2:DescribeSubnets" - ] - - condition { - test = "StringEquals" - variable = "aws:RequestedRegion" - values = [local.region] - } - } - - statement { - sid = "AllowSSMReadActions" - resources = coalescelist(var.ami_id_ssm_parameter_arns, ["arn:${local.partition}:ssm:${local.region}::parameter/aws/service/*"]) - actions = ["ssm:GetParameter"] - } - - statement { - sid = "AllowPricingReadActions" - resources = ["*"] - actions = ["pricing:GetProducts"] - } - - dynamic "statement" { - for_each = local.enable_spot_termination ? [1] : [] - - content { - sid = "AllowInterruptionQueueActions" - resources = [try(aws_sqs_queue.this[0].arn, null)] - actions = [ - "sqs:DeleteMessage", - "sqs:GetQueueAttributes", - "sqs:GetQueueUrl", - "sqs:ReceiveMessage" - ] - } - } - - statement { - sid = "AllowPassingInstanceRole" - resources = var.create_node_iam_role ? [aws_iam_role.node[0].arn] : [var.node_iam_role_arn] - actions = ["iam:PassRole"] - - condition { - test = "StringEquals" - variable = "iam:PassedToService" - values = ["ec2.amazonaws.com"] - } - } - - statement { - sid = "AllowScopedInstanceProfileCreationActions" - resources = ["*"] - actions = ["iam:CreateInstanceProfile"] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "aws:RequestTag/topology.kubernetes.io/region" - values = [local.region] - } - - condition { - test = "StringLike" - variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" - values = ["*"] - } - } - - statement { - sid = "AllowScopedInstanceProfileTagActions" - resources = ["*"] - actions = ["iam:TagInstanceProfile"] - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/topology.kubernetes.io/region" - values = [local.region] - } - - condition { - test = "StringEquals" - variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/topology.kubernetes.io/region" - values = [local.region] - } - - condition { - test = "StringLike" - variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" - values = ["*"] - } - - condition { - test = "StringLike" - variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" - values = ["*"] - } - } - - statement { - sid = "AllowScopedInstanceProfileActions" - resources = ["*"] - actions = [ - "iam:AddRoleToInstanceProfile", - "iam:RemoveRoleFromInstanceProfile", - "iam:DeleteInstanceProfile" - ] - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/topology.kubernetes.io/region" - values = [local.region] - } - - condition { - test = "StringLike" - variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" - values = ["*"] - } - } - - statement { - sid = "AllowInstanceProfileReadActions" - resources = ["*"] - actions = ["iam:GetInstanceProfile"] - } - - statement { - sid = "AllowAPIServerEndpointDiscovery" - resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] - actions = ["eks:DescribeCluster"] - } - - dynamic "statement" { - for_each = var.iam_policy_statements - - content { - sid = try(statement.value.sid, null) - actions = try(statement.value.actions, null) - not_actions = try(statement.value.not_actions, null) - effect = try(statement.value.effect, null) - resources = try(statement.value.resources, null) - not_resources = try(statement.value.not_resources, null) - - dynamic "principals" { - for_each = try(statement.value.principals, []) - - content { - type = principals.value.type - identifiers = principals.value.identifiers - } - } - - dynamic "not_principals" { - for_each = try(statement.value.not_principals, []) - - content { - type = not_principals.value.type - identifiers = not_principals.value.identifiers - } - } - - dynamic "condition" { - for_each = try(statement.value.conditions, []) - - content { - test = condition.value.test - values = condition.value.values - variable = condition.value.variable - } - } - } - } + source_policy_documents = var.enable_v1_permissions ? [data.aws_iam_policy_document.v1[0].json] : [data.aws_iam_policy_document.v033[0].json] } resource "aws_iam_policy" "controller" { diff --git a/modules/karpenter/policy.tf b/modules/karpenter/policy.tf new file mode 100644 index 0000000000..456a27f417 --- /dev/null +++ b/modules/karpenter/policy.tf @@ -0,0 +1,749 @@ +################################################################################ +# v0.33.x - v0.37.x Controller IAM Policy +################################################################################ + +data "aws_iam_policy_document" "v033" { + count = local.create_iam_role ? 1 : 0 + + statement { + sid = "AllowScopedEC2InstanceActions" + resources = [ + "arn:${local.partition}:ec2:*::image/*", + "arn:${local.partition}:ec2:*::snapshot/*", + "arn:${local.partition}:ec2:*:*:spot-instances-request/*", + "arn:${local.partition}:ec2:*:*:security-group/*", + "arn:${local.partition}:ec2:*:*:subnet/*", + "arn:${local.partition}:ec2:*:*:launch-template/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + } + + statement { + sid = "AllowScopedEC2InstanceActionsWithTags" + resources = [ + "arn:${local.partition}:ec2:*:*:fleet/*", + "arn:${local.partition}:ec2:*:*:instance/*", + "arn:${local.partition}:ec2:*:*:volume/*", + "arn:${local.partition}:ec2:*:*:network-interface/*", + "arn:${local.partition}:ec2:*:*:launch-template/*", + "arn:${local.partition}:ec2:*:*:spot-instances-request/*", + ] + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate" + ] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceCreationTagging" + resources = [ + "arn:${local.partition}:ec2:*:*:fleet/*", + "arn:${local.partition}:ec2:*:*:instance/*", + "arn:${local.partition}:ec2:*:*:volume/*", + "arn:${local.partition}:ec2:*:*:network-interface/*", + "arn:${local.partition}:ec2:*:*:launch-template/*", + "arn:${local.partition}:ec2:*:*:spot-instances-request/*", + ] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + values = [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate", + ] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceTagging" + resources = ["arn:${local.partition}:ec2:*:*:instance/*"] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = [ + "karpenter.sh/nodeclaim", + "Name", + ] + } + } + + statement { + sid = "AllowScopedDeletion" + resources = [ + "arn:${local.partition}:ec2:*:*:instance/*", + "arn:${local.partition}:ec2:*:*:launch-template/*" + ] + + actions = [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowRegionalReadActions" + resources = ["*"] + actions = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ] + + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [local.region] + } + } + + statement { + sid = "AllowSSMReadActions" + resources = coalescelist(var.ami_id_ssm_parameter_arns, ["arn:${local.partition}:ssm:${local.region}::parameter/aws/service/*"]) + actions = ["ssm:GetParameter"] + } + + statement { + sid = "AllowPricingReadActions" + resources = ["*"] + actions = ["pricing:GetProducts"] + } + + dynamic "statement" { + for_each = local.enable_spot_termination ? [1] : [] + + content { + sid = "AllowInterruptionQueueActions" + resources = [try(aws_sqs_queue.this[0].arn, null)] + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] + } + } + + statement { + sid = "AllowPassingInstanceRole" + resources = var.create_node_iam_role ? [aws_iam_role.node[0].arn] : [var.node_iam_role_arn] + actions = ["iam:PassRole"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["ec2.amazonaws.com"] + } + } + + statement { + sid = "AllowScopedInstanceProfileCreationActions" + resources = ["*"] + actions = ["iam:CreateInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileTagActions" + resources = ["*"] + actions = ["iam:TagInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileActions" + resources = ["*"] + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowInstanceProfileReadActions" + resources = ["*"] + actions = ["iam:GetInstanceProfile"] + } + + statement { + sid = "AllowAPIServerEndpointDiscovery" + resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] + actions = ["eks:DescribeCluster"] + } + + dynamic "statement" { + for_each = var.iam_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +################################################################################ +# v1.0.x Controller IAM Policy +################################################################################ + +data "aws_iam_policy_document" "v1" { + count = local.create_iam_role ? 1 : 0 + + statement { + sid = "AllowScopedEC2InstanceAccessActions" + resources = [ + "arn:${local.partition}:ec2:${local.region}::image/*", + "arn:${local.partition}:ec2:${local.region}::snapshot/*", + "arn:${local.partition}:ec2:${local.region}:*:security-group/*", + "arn:${local.partition}:ec2:${local.region}:*:subnet/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + } + + statement { + sid = "AllowScopedEC2LaunchTemplateAccessActions" + resources = [ + "arn:${local.partition}:ec2:${local.region}:*:launch-template/*" + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedEC2InstanceActionsWithTags" + resources = [ + "arn:${local.partition}:ec2:${local.region}:*:fleet/*", + "arn:${local.partition}:ec2:${local.region}:*:instance/*", + "arn:${local.partition}:ec2:${local.region}:*:volume/*", + "arn:${local.partition}:ec2:${local.region}:*:network-interface/*", + "arn:${local.partition}:ec2:${local.region}:*:launch-template/*", + "arn:${local.partition}:ec2:${local.region}:*:spot-instances-request/*", + ] + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate" + ] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_name] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceCreationTagging" + resources = [ + "arn:${local.partition}:ec2:${local.region}:*:fleet/*", + "arn:${local.partition}:ec2:${local.region}:*:instance/*", + "arn:${local.partition}:ec2:${local.region}:*:volume/*", + "arn:${local.partition}:ec2:${local.region}:*:network-interface/*", + "arn:${local.partition}:ec2:${local.region}:*:launch-template/*", + "arn:${local.partition}:ec2:${local.region}:*:spot-instances-request/*", + ] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_name] + } + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + values = [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate", + ] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceTagging" + resources = ["arn:${local.partition}:ec2:${local.region}:*:instance/*"] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + + condition { + test = "StringEqualsIfExists" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_name] + } + + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = [ + "eks:eks-cluster-name", + "karpenter.sh/nodeclaim", + "Name", + ] + } + } + + statement { + sid = "AllowScopedDeletion" + resources = [ + "arn:${local.partition}:ec2:${local.region}:*:instance/*", + "arn:${local.partition}:ec2:${local.region}:*:launch-template/*" + ] + + actions = [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowRegionalReadActions" + resources = ["*"] + actions = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ] + + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [local.region] + } + } + + statement { + sid = "AllowSSMReadActions" + resources = coalescelist(var.ami_id_ssm_parameter_arns, ["arn:${local.partition}:ssm:${local.region}::parameter/aws/service/*"]) + actions = ["ssm:GetParameter"] + } + + statement { + sid = "AllowPricingReadActions" + resources = ["*"] + actions = ["pricing:GetProducts"] + } + + dynamic "statement" { + for_each = local.enable_spot_termination ? [1] : [] + + content { + sid = "AllowInterruptionQueueActions" + resources = [try(aws_sqs_queue.this[0].arn, null)] + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] + } + } + + statement { + sid = "AllowPassingInstanceRole" + resources = var.create_node_iam_role ? [aws_iam_role.node[0].arn] : [var.node_iam_role_arn] + actions = ["iam:PassRole"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["ec2.amazonaws.com"] + } + } + + statement { + sid = "AllowScopedInstanceProfileCreationActions" + resources = ["arn:${local.partition}:iam::${local.account_id}:instance-profile/*"] + actions = ["iam:CreateInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileTagActions" + resources = ["arn:${local.partition}:iam::${local.account_id}:instance-profile/*"] + actions = ["iam:TagInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileActions" + resources = ["arn:${local.partition}:iam::${local.account_id}:instance-profile/*"] + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowInstanceProfileReadActions" + resources = ["arn:${local.partition}:iam::${local.account_id}:instance-profile/*"] + actions = ["iam:GetInstanceProfile"] + } + + statement { + sid = "AllowAPIServerEndpointDiscovery" + resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] + actions = ["eks:DescribeCluster"] + } + + dynamic "statement" { + for_each = var.iam_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 96d898b245..a2c307fc05 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -116,6 +116,13 @@ variable "enable_pod_identity" { default = true } +# TODO - make v1 permssions the default policy at next breaking change +variable "enable_v1_permissions" { + description = "Determines whether to enable permissions suitable for v1+ (`true`) or for v0.33.x-v0.37.x (`false`)" + type = bool + default = false +} + ################################################################################ # IAM Role for Service Account (IRSA) ################################################################################ From c60b70fbc80606eb4ed8cf47063ac6ed0d8dd435 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 19 Aug 2024 19:32:05 +0000 Subject: [PATCH 818/962] chore(release): version 20.24.0 [skip ci] ## [20.24.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.23.0...v20.24.0) (2024-08-19) ### Features * Add support for Karpenter v1 controller IAM role permissions ([#3126](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3126)) ([e317651](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e31765153570631c1978e11cfd1d28e5fc349d8f)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be11e0f6d7..178d0d672c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.24.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.23.0...v20.24.0) (2024-08-19) + + +### Features + +* Add support for Karpenter v1 controller IAM role permissions ([#3126](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3126)) ([e317651](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e31765153570631c1978e11cfd1d28e5fc349d8f)) + ## [20.23.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.22.1...v20.23.0) (2024-08-09) From 71d19472f294d6cbb67ffbfb081fd54cad32551a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 13 Sep 2024 16:31:42 -0500 Subject: [PATCH 819/962] chore: Remove outposts example (#3154) --- .pre-commit-config.yaml | 2 +- README.md | 5 +- examples/outposts/README.md | 122 ---------- examples/outposts/main.tf | 133 ----------- examples/outposts/outputs.tf | 226 ------------------- examples/outposts/prerequisites/main.tf | 149 ------------ examples/outposts/prerequisites/outputs.tf | 4 - examples/outposts/prerequisites/variables.tf | 5 - examples/outposts/prerequisites/versions.tf | 10 - examples/outposts/variables.tf | 5 - examples/outposts/versions.tf | 14 -- 11 files changed, 3 insertions(+), 672 deletions(-) delete mode 100644 examples/outposts/README.md delete mode 100644 examples/outposts/main.tf delete mode 100644 examples/outposts/outputs.tf delete mode 100644 examples/outposts/prerequisites/main.tf delete mode 100644 examples/outposts/prerequisites/outputs.tf delete mode 100644 examples/outposts/prerequisites/variables.tf delete mode 100644 examples/outposts/prerequisites/versions.tf delete mode 100644 examples/outposts/variables.tf delete mode 100644 examples/outposts/versions.tf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b73ed98715..28c6063b6d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.92.2 + rev: v1.95.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index 6aad481af7..fa77df6251 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,6 @@ module "eks" { - [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-managed-node-group): EKS Cluster using EKS managed node groups - [Karpenter](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/karpenter): EKS Cluster with [Karpenter](https://karpenter.sh/) provisioned for intelligent data plane management -- [Outposts](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/outposts): EKS local cluster provisioned on [AWS Outposts](https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts.html) - [Self Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self-managed-node-group): EKS Cluster using self-managed node groups ## Contributing @@ -170,7 +169,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple - [Code of Conduct](https://github.com/terraform-aws-modules/.github/blob/master/CODE_OF_CONDUCT.md) - [Contributing Guide](https://github.com/terraform-aws-modules/.github/blob/master/CONTRIBUTING.md) - + ## Requirements | Name | Version | @@ -365,7 +364,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | | [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | - + ## License diff --git a/examples/outposts/README.md b/examples/outposts/README.md deleted file mode 100644 index 5057fc4a5e..0000000000 --- a/examples/outposts/README.md +++ /dev/null @@ -1,122 +0,0 @@ -# EKS on Outposts Example - -Configuration in this directory creates an AWS EKS local cluster on AWS Outposts - -See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts.html) for further details. - -Note: This example requires an an AWS Outpost to provision. - -## Usage - -To run this example you need to: - -1. Deploy the remote host where the cluster will be provisioned from. The remote host is required since only private access is permitted to clusters created on Outposts. If you have access to the network where Outposts are provisioned (VPN, etc.), you can skip this step: - -```bash -$ cd prerequisites -$ terraform init -$ terraform plan -$ terraform apply --auto-approve -``` - -2. If provisioning using the remote host deployed in step 1, connect to the remote host using SSM. Note, you will need to have the [SSM plugin for the AWS CLI installed](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html). You can use the output generated by step 1 to connect: - -```bash -$ aws ssm start-session --region --target -``` - -3. Once connected to the remote host, navigate to the cloned project example directory and deploy the example: - -```bash -$ cd $HOME/terraform-aws-eks/examples/outposts -$ terraform init -$ terraform plan -$ terraform apply --auto-approve -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - -```bash -terraform destroy --auto-approve -``` - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.61 | -| [kubernetes](#requirement\_kubernetes) | >= 2.20 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 5.61 | -| [kubernetes](#provider\_kubernetes) | >= 2.20 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | ../.. | n/a | - -## Resources - -| Name | Type | -|------|------| -| [kubernetes_storage_class_v1.this](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/storage_class_v1) | resource | -| [aws_outposts_outpost_instance_types.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/outposts_outpost_instance_types) | data source | -| [aws_outposts_outposts.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/outposts_outposts) | data source | -| [aws_subnet.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | -| [aws_subnets.lookup](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | -| [aws_subnets.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | -| [aws_vpc.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [region](#input\_region) | The AWS region to deploy into (e.g. us-east-1) | `string` | `"us-west-2"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | -| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | -| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | -| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | -| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | -| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | -| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | -| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | -| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | -| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | -| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | -| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | -| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | -| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | -| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | -| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | -| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | -| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | -| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | -| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | -| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | -| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | -| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | -| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | -| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | -| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | -| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | -| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | -| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | -| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | -| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | -| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | - diff --git a/examples/outposts/main.tf b/examples/outposts/main.tf deleted file mode 100644 index c7835971f7..0000000000 --- a/examples/outposts/main.tf +++ /dev/null @@ -1,133 +0,0 @@ -provider "aws" { - region = var.region -} - -locals { - name = "ex-${basename(path.cwd)}" - cluster_version = "1.30" - - outpost_arn = element(tolist(data.aws_outposts_outposts.this.arns), 0) - instance_type = element(tolist(data.aws_outposts_outpost_instance_types.this.instance_types), 0) - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_version = local.cluster_version - - cluster_endpoint_public_access = false # Not available on Outpost - cluster_endpoint_private_access = true - - # Gives Terraform identity admin access to cluster which will - # allow deploying resources (EBS storage class) into the cluster - enable_cluster_creator_admin_permissions = true - - vpc_id = data.aws_vpc.this.id - subnet_ids = data.aws_subnets.this.ids - - outpost_config = { - control_plane_instance_type = local.instance_type - outpost_arns = [local.outpost_arn] - } - - # Extend cluster security group rules - cluster_security_group_additional_rules = { - ingress_vpc_https = { - description = "Remote host to control plane" - protocol = "tcp" - from_port = 443 - to_port = 443 - type = "ingress" - cidr_blocks = [data.aws_vpc.this.cidr_block] - } - } - - self_managed_node_groups = { - outpost = { - name = local.name - - min_size = 2 - max_size = 5 - desired_size = 3 - instance_type = local.instance_type - - # Additional information is required to join local clusters to EKS - bootstrap_extra_args = <<-EOT - --enable-local-outpost true --cluster-id ${module.eks.cluster_id} --container-runtime containerd - EOT - } - } - - tags = local.tags -} - -resource "kubernetes_storage_class_v1" "this" { - metadata { - name = "ebs-sc" - annotations = { - "storageclass.kubernetes.io/is-default-class" = "true" - } - } - - storage_provisioner = "ebs.csi.aws.com" - volume_binding_mode = "WaitForFirstConsumer" - allow_volume_expansion = true - - parameters = { - type = "gp2" - encrypted = "true" - } -} - -################################################################################ -# Supporting Resources -################################################################################ - -data "aws_outposts_outposts" "this" {} - -data "aws_outposts_outpost_instance_types" "this" { - arn = local.outpost_arn -} - -# This just grabs the first Outpost and returns its subnets -data "aws_subnets" "lookup" { - filter { - name = "outpost-arn" - values = [local.outpost_arn] - } -} - -# This grabs a single subnet to reverse lookup those that belong to same VPC -# This is whats used for the cluster -data "aws_subnet" "this" { - id = element(tolist(data.aws_subnets.lookup.ids), 0) -} - -# These are subnets for the Outpost and restricted to the same VPC -# This is whats used for the cluster -data "aws_subnets" "this" { - filter { - name = "outpost-arn" - values = [local.outpost_arn] - } - - filter { - name = "vpc-id" - values = [data.aws_subnet.this.vpc_id] - } -} - -data "aws_vpc" "this" { - id = data.aws_subnet.this.vpc_id -} diff --git a/examples/outposts/outputs.tf b/examples/outposts/outputs.tf deleted file mode 100644 index 9357464c29..0000000000 --- a/examples/outposts/outputs.tf +++ /dev/null @@ -1,226 +0,0 @@ -################################################################################ -# Cluster -################################################################################ - -output "cluster_arn" { - description = "The Amazon Resource Name (ARN) of the cluster" - value = module.eks.cluster_arn -} - -output "cluster_certificate_authority_data" { - description = "Base64 encoded certificate data required to communicate with the cluster" - value = module.eks.cluster_certificate_authority_data -} - -output "cluster_endpoint" { - description = "Endpoint for your Kubernetes API server" - value = module.eks.cluster_endpoint -} - -output "cluster_id" { - description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" - value = module.eks.cluster_id -} - -output "cluster_name" { - description = "The name of the EKS cluster" - value = module.eks.cluster_name -} - -output "cluster_oidc_issuer_url" { - description = "The URL on the EKS cluster for the OpenID Connect identity provider" - value = module.eks.cluster_oidc_issuer_url -} - -output "cluster_dualstack_oidc_issuer_url" { - description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" - value = module.eks.cluster_dualstack_oidc_issuer_url -} - -output "cluster_platform_version" { - description = "Platform version for the cluster" - value = module.eks.cluster_platform_version -} - -output "cluster_status" { - description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" - value = module.eks.cluster_status -} - -output "cluster_primary_security_group_id" { - description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" - value = module.eks.cluster_primary_security_group_id -} - -output "cluster_service_cidr" { - description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" - value = module.eks.cluster_service_cidr -} - -output "cluster_ip_family" { - description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" - value = module.eks.cluster_ip_family -} - -################################################################################ -# Access Entry -################################################################################ - -output "access_entries" { - description = "Map of access entries created and their attributes" - value = module.eks.access_entries -} - -################################################################################ -# KMS Key -################################################################################ - -output "kms_key_arn" { - description = "The Amazon Resource Name (ARN) of the key" - value = module.eks.kms_key_arn -} - -output "kms_key_id" { - description = "The globally unique identifier for the key" - value = module.eks.kms_key_id -} - -output "kms_key_policy" { - description = "The IAM resource policy set on the key" - value = module.eks.kms_key_policy -} - -################################################################################ -# Security Group -################################################################################ - -output "cluster_security_group_arn" { - description = "Amazon Resource Name (ARN) of the cluster security group" - value = module.eks.cluster_security_group_arn -} - -output "cluster_security_group_id" { - description = "ID of the cluster security group" - value = module.eks.cluster_security_group_id -} - -################################################################################ -# Node Security Group -################################################################################ - -output "node_security_group_arn" { - description = "Amazon Resource Name (ARN) of the node shared security group" - value = module.eks.node_security_group_arn -} - -output "node_security_group_id" { - description = "ID of the node shared security group" - value = module.eks.node_security_group_id -} - -################################################################################ -# IRSA -################################################################################ - -output "oidc_provider" { - description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" - value = module.eks.oidc_provider -} - -output "oidc_provider_arn" { - description = "The ARN of the OIDC Provider if `enable_irsa = true`" - value = module.eks.oidc_provider_arn -} - -output "cluster_tls_certificate_sha1_fingerprint" { - description = "The SHA1 fingerprint of the public key of the cluster's certificate" - value = module.eks.cluster_tls_certificate_sha1_fingerprint -} - -################################################################################ -# IAM Role -################################################################################ - -output "cluster_iam_role_name" { - description = "IAM role name of the EKS cluster" - value = module.eks.cluster_iam_role_name -} - -output "cluster_iam_role_arn" { - description = "IAM role ARN of the EKS cluster" - value = module.eks.cluster_iam_role_arn -} - -output "cluster_iam_role_unique_id" { - description = "Stable and unique string identifying the IAM role" - value = module.eks.cluster_iam_role_unique_id -} - -################################################################################ -# EKS Addons -################################################################################ - -output "cluster_addons" { - description = "Map of attribute maps for all EKS cluster addons enabled" - value = module.eks.cluster_addons -} - -################################################################################ -# EKS Identity Provider -################################################################################ - -output "cluster_identity_providers" { - description = "Map of attribute maps for all EKS identity providers enabled" - value = module.eks.cluster_identity_providers -} - -################################################################################ -# CloudWatch Log Group -################################################################################ - -output "cloudwatch_log_group_name" { - description = "Name of cloudwatch log group created" - value = module.eks.cloudwatch_log_group_name -} - -output "cloudwatch_log_group_arn" { - description = "Arn of cloudwatch log group created" - value = module.eks.cloudwatch_log_group_arn -} - -################################################################################ -# Fargate Profile -################################################################################ - -output "fargate_profiles" { - description = "Map of attribute maps for all EKS Fargate Profiles created" - value = module.eks.fargate_profiles -} - -################################################################################ -# EKS Managed Node Group -################################################################################ - -output "eks_managed_node_groups" { - description = "Map of attribute maps for all EKS managed node groups created" - value = module.eks.eks_managed_node_groups -} - -output "eks_managed_node_groups_autoscaling_group_names" { - description = "List of the autoscaling group names created by EKS managed node groups" - value = module.eks.eks_managed_node_groups_autoscaling_group_names -} - -################################################################################ -# Self Managed Node Group -################################################################################ - -output "self_managed_node_groups" { - description = "Map of attribute maps for all self managed node groups created" - value = module.eks.self_managed_node_groups -} - -output "self_managed_node_groups_autoscaling_group_names" { - description = "List of the autoscaling group names created by self-managed node groups" - value = module.eks.self_managed_node_groups_autoscaling_group_names -} diff --git a/examples/outposts/prerequisites/main.tf b/examples/outposts/prerequisites/main.tf deleted file mode 100644 index 1a1dd18a4b..0000000000 --- a/examples/outposts/prerequisites/main.tf +++ /dev/null @@ -1,149 +0,0 @@ -provider "aws" { - region = var.region -} - -locals { - name = "ex-${basename(path.cwd)}" - - terraform_version = "1.3.6" - - outpost_arn = element(tolist(data.aws_outposts_outposts.this.arns), 0) - instance_type = element(tolist(data.aws_outposts_outpost_instance_types.this.instance_types), 0) - - tags = { - Example = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# Pre-Requisites -################################################################################ - -module "ssm_bastion_ec2" { - source = "terraform-aws-modules/ec2-instance/aws" - version = "~> 5.5" - - name = "${local.name}-bastion" - - create_iam_instance_profile = true - iam_role_policies = { - AdministratorAccess = "arn:aws:iam::aws:policy/AdministratorAccess" - } - - instance_type = local.instance_type - - user_data = <<-EOT - #!/bin/bash - - # Add ssm-user since it won't exist until first login - adduser -m ssm-user - tee /etc/sudoers.d/ssm-agent-users <<'EOF' - # User rules for ssm-user - ssm-user ALL=(ALL) NOPASSWD:ALL - EOF - chmod 440 /etc/sudoers.d/ssm-agent-users - - cd /home/ssm-user - - # Install git to clone repo - yum install git -y - - # Install Terraform - curl -sSO https://releases.hashicorp.com/terraform/${local.terraform_version}/terraform_${local.terraform_version}_linux_amd64.zip - sudo unzip -qq terraform_${local.terraform_version}_linux_amd64.zip terraform -d /usr/bin/ - rm terraform_${local.terraform_version}_linux_amd64.zip 2> /dev/null - - # Install kubectl - curl -LO https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - - # Remove default awscli which is v1 - we want latest v2 - yum remove awscli -y - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -qq awscliv2.zip - ./aws/install - - # Clone repo - git clone https://github.com/terraform-aws-modules/terraform-aws-eks.git \ - && cd /home/ssm-user/terraform-aws-eks - - chown -R ssm-user:ssm-user /home/ssm-user/ - EOT - - vpc_security_group_ids = [module.bastion_security_group.security_group_id] - subnet_id = element(data.aws_subnets.this.ids, 0) - - tags = local.tags -} - -module "bastion_security_group" { - source = "terraform-aws-modules/security-group/aws" - version = "~> 5.0" - - name = "${local.name}-bastion" - description = "Security group to allow provisioning ${local.name} EKS local cluster on Outposts" - vpc_id = data.aws_vpc.this.id - - ingress_with_cidr_blocks = [ - { - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = data.aws_vpc.this.cidr_block - }, - ] - egress_with_cidr_blocks = [ - { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = "0.0.0.0/0" - }, - ] - - tags = local.tags -} - -################################################################################ -# Supporting Resources -################################################################################ - -data "aws_outposts_outposts" "this" {} - -data "aws_outposts_outpost_instance_types" "this" { - arn = local.outpost_arn -} - -# This just grabs the first Outpost and returns its subnets -data "aws_subnets" "lookup" { - filter { - name = "outpost-arn" - values = [local.outpost_arn] - } -} - -# This grabs a single subnet to reverse lookup those that belong to same VPC -# This is whats used for the cluster -data "aws_subnet" "this" { - id = element(tolist(data.aws_subnets.lookup.ids), 0) -} - -# These are subnets for the Outpost and restricted to the same VPC -# This is whats used for the cluster -data "aws_subnets" "this" { - filter { - name = "outpost-arn" - values = [local.outpost_arn] - } - - filter { - name = "vpc-id" - values = [data.aws_subnet.this.vpc_id] - } -} - -data "aws_vpc" "this" { - id = data.aws_subnet.this.vpc_id -} diff --git a/examples/outposts/prerequisites/outputs.tf b/examples/outposts/prerequisites/outputs.tf deleted file mode 100644 index f2ff81ab70..0000000000 --- a/examples/outposts/prerequisites/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "ssm_start_session" { - description = "SSM start session command to connect to remote host created" - value = "aws ssm start-session --region ${var.region} --target ${module.ssm_bastion_ec2.id}" -} diff --git a/examples/outposts/prerequisites/variables.tf b/examples/outposts/prerequisites/variables.tf deleted file mode 100644 index 47945c8501..0000000000 --- a/examples/outposts/prerequisites/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "region" { - description = "The AWS region to deploy into (e.g. us-east-1)" - type = string - default = "us-west-2" -} diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf deleted file mode 100644 index 5bfe6da389..0000000000 --- a/examples/outposts/prerequisites/versions.tf +++ /dev/null @@ -1,10 +0,0 @@ -terraform { - required_version = ">= 1.3.2" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.61" - } - } -} diff --git a/examples/outposts/variables.tf b/examples/outposts/variables.tf deleted file mode 100644 index 47945c8501..0000000000 --- a/examples/outposts/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "region" { - description = "The AWS region to deploy into (e.g. us-east-1)" - type = string - default = "us-west-2" -} diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf deleted file mode 100644 index 9836b3a468..0000000000 --- a/examples/outposts/versions.tf +++ /dev/null @@ -1,14 +0,0 @@ -terraform { - required_version = ">= 1.3.2" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.61" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.20" - } - } -} From 47ab3eb884ab243a99322998445127ea6802fcaf Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 16 Sep 2024 10:36:15 -0500 Subject: [PATCH 820/962] fix: Correct Karpenter EC2 service principal DNS suffix in non-commercial regions (#3157) --- .pre-commit-config.yaml | 2 +- modules/karpenter/main.tf | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 28c6063b6d..9530aa8f8a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.95.0 + rev: v1.96.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index fc4a5dca57..d03dfa49f8 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -4,6 +4,7 @@ data "aws_caller_identity" "current" {} locals { account_id = data.aws_caller_identity.current.account_id + dns_suffix = data.aws_partition.current.dns_suffix partition = data.aws_partition.current.partition region = data.aws_region.current.name } @@ -286,7 +287,7 @@ data "aws_iam_policy_document" "node_assume_role" { principals { type = "Service" - identifiers = ["ec2.amazonaws.com"] + identifiers = ["ec2.${local.dns_suffix}"] } } } From 2965d99e1ecca710bbdf8fbccb208d042239e8e2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 16 Sep 2024 15:36:45 +0000 Subject: [PATCH 821/962] chore(release): version 20.24.1 [skip ci] ## [20.24.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.0...v20.24.1) (2024-09-16) ### Bug Fixes * Correct Karpenter EC2 service principal DNS suffix in non-commercial regions ([#3157](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3157)) ([47ab3eb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/47ab3eb884ab243a99322998445127ea6802fcaf)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 178d0d672c..cfeb4d8e90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.24.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.0...v20.24.1) (2024-09-16) + + +### Bug Fixes + +* Correct Karpenter EC2 service principal DNS suffix in non-commercial regions ([#3157](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3157)) ([47ab3eb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/47ab3eb884ab243a99322998445127ea6802fcaf)) + ## [20.24.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.23.0...v20.24.0) (2024-08-19) From 8b90872983b9c349ff2e0a71678d687dc32ed626 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 21 Sep 2024 09:45:50 -0500 Subject: [PATCH 822/962] fix: Remove deprecated `inline_policy` from cluster role (#3163) --- .github/workflows/pre-commit.yml | 4 ++-- .pre-commit-config.yaml | 2 +- README.md | 6 +++--- examples/karpenter/README.md | 4 ++-- main.tf | 23 ----------------------- modules/_user_data/README.md | 8 ++++---- modules/aws-auth/README.md | 4 ++-- modules/eks-managed-node-group/README.md | 14 +++++++------- modules/fargate-profile/README.md | 4 ++-- modules/karpenter/README.md | 6 +++--- modules/self-managed-node-group/README.md | 14 +++++++------- tests/eks-managed-node-group/README.md | 4 ++-- tests/fargate-profile/README.md | 4 ++-- tests/self-managed-node-group/README.md | 4 ++-- tests/user-data/README.md | 4 ++-- 15 files changed, 41 insertions(+), 64 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index c2632d1a44..71878b4351 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -7,8 +7,8 @@ on: - master env: - TERRAFORM_DOCS_VERSION: v0.16.0 - TFLINT_VERSION: v0.50.3 + TERRAFORM_DOCS_VERSION: v0.19.0 + TFLINT_VERSION: v0.53.0 jobs: collectInputs: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9530aa8f8a..55b241248d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.96.0 + rev: v1.96.1 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index fa77df6251..37008e7824 100644 --- a/README.md +++ b/README.md @@ -243,8 +243,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | | [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no | | [cluster\_addons\_timeouts](#input\_cluster\_addons\_timeouts) | Create, update, and delete timeout configurations for the cluster addons | `map(string)` | `{}` | no | -| [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | -| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster. To disable secret encryption, set this value to `{}` | `any` |
{
"resources": [
"secrets"
]
}
| no | +| [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | +| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster. To disable secret encryption, set this value to `{}` | `any` |
{
"resources": [
"secrets"
]
}
| no | | [cluster\_encryption\_policy\_description](#input\_cluster\_encryption\_policy\_description) | Description of the cluster encryption policy created | `string` | `"Cluster encryption policy to allow cluster role to utilize CMK provided"` | no | | [cluster\_encryption\_policy\_name](#input\_cluster\_encryption\_policy\_name) | Name to use on cluster encryption policy created | `string` | `null` | no | | [cluster\_encryption\_policy\_path](#input\_cluster\_encryption\_policy\_path) | Cluster encryption policy path | `string` | `null` | no | @@ -252,7 +252,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_encryption\_policy\_use\_name\_prefix](#input\_cluster\_encryption\_policy\_use\_name\_prefix) | Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix | `bool` | `true` | no | | [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `true` | no | | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `false` | no | -| [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | +| [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | | [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `"ipv4"` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 62291cdc07..3ec2d6c2e7 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -83,7 +83,7 @@ terraform destroy --auto-approve Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - + ## Requirements | Name | Version | @@ -176,4 +176,4 @@ No inputs. | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | | [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | - + diff --git a/main.tf b/main.tf index 1846a1da05..b891b26d0e 100644 --- a/main.tf +++ b/main.tf @@ -411,29 +411,6 @@ resource "aws_iam_role" "this" { permissions_boundary = var.iam_role_permissions_boundary force_detach_policies = true - # https://github.com/terraform-aws-modules/terraform-aws-eks/issues/920 - # Resources running on the cluster are still generating logs when destroying the module resources - # which results in the log group being re-created even after Terraform destroys it. Removing the - # ability for the cluster role to create the log group prevents this log group from being re-created - # outside of Terraform due to services still generating logs during destroy process - dynamic "inline_policy" { - for_each = var.create_cloudwatch_log_group ? [1] : [] - content { - name = local.iam_role_name - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Action = ["logs:CreateLogGroup"] - Effect = "Deny" - Resource = "*" - }, - ] - }) - } - } - tags = merge(var.tags, var.iam_role_tags) } diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index e5207d9443..09c4b98a4e 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -4,7 +4,7 @@ Configuration in this directory renders the appropriate user data for the given See [`examples/user_data/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) for various examples using this module. - + ## Requirements | Name | Version | @@ -39,8 +39,8 @@ No modules. | [additional\_cluster\_dns\_ips](#input\_additional\_cluster\_dns\_ips) | Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*` | `list(string)` | `[]` | no | | [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `null` | no | | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | -| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | -| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | @@ -61,4 +61,4 @@ No modules. |------|-------------| | [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023, or `windows | | [user\_data](#output\_user\_data) | Base64 encoded user data rendered for the provided inputs | - + diff --git a/modules/aws-auth/README.md b/modules/aws-auth/README.md index 5ba490b7f1..d4b97f14ca 100644 --- a/modules/aws-auth/README.md +++ b/modules/aws-auth/README.md @@ -39,7 +39,7 @@ module "eks" { ## Usage - + ## Requirements | Name | Version | @@ -78,4 +78,4 @@ No modules. ## Outputs No outputs. - + diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 96b6c4f290..de0c95b103 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -58,7 +58,7 @@ module "eks_managed_node_group" { } ``` - + ## Requirements | Name | Version | @@ -110,8 +110,8 @@ module "eks_managed_node_group" { | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | | [capacity\_type](#input\_capacity\_type) | Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT` | `string` | `"ON_DEMAND"` | no | -| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | -| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | @@ -164,7 +164,7 @@ module "eks_managed_node_group" { | [license\_specifications](#input\_license\_specifications) | A map of license specifications to associate with | `any` | `{}` | no | | [maintenance\_options](#input\_maintenance\_options) | The maintenance options for the instance | `any` | `{}` | no | | [max\_size](#input\_max\_size) | Maximum number of instances/nodes | `number` | `3` | no | -| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | +| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | | [min\_size](#input\_min\_size) | Minimum number of instances/nodes | `number` | `0` | no | | [name](#input\_name) | Name of the EKS managed node group | `string` | `""` | no | | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | @@ -179,11 +179,11 @@ module "eks_managed_node_group" { | [remote\_access](#input\_remote\_access) | Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false` | `any` | `{}` | no | | [schedules](#input\_schedules) | Map of autoscaling group schedule to create | `map(any)` | `{}` | no | | [subnet\_ids](#input\_subnet\_ids) | Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME` | `list(string)` | `null` | no | -| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | +| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [taints](#input\_taints) | The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group | `any` | `{}` | no | | [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the node group | `map(string)` | `{}` | no | -| [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates | `map(string)` |
{
"max_unavailable_percentage": 33
}
| no | +| [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates | `map(string)` |
{
"max_unavailable_percentage": 33
}
| no | | [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update the launch templates default version on each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | | [use\_custom\_launch\_template](#input\_use\_custom\_launch\_template) | Determines whether to use a custom launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | | [use\_latest\_ami\_release\_version](#input\_use\_latest\_ami\_release\_version) | Determines whether to use the latest AMI release version for the given `ami_type` (except for `CUSTOM`). Note: `ami_type` and `cluster_version` must be supplied in order to enable this feature | `bool` | `false` | no | @@ -211,4 +211,4 @@ module "eks_managed_node_group" { | [node\_group\_status](#output\_node\_group\_status) | Status of the EKS Node Group | | [node\_group\_taints](#output\_node\_group\_taints) | List of objects containing information about taints applied to the node group | | [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | - + diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index b1ee9e0ade..e6be895ebe 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -23,7 +23,7 @@ module "fargate_profile" { } ``` - + ## Requirements | Name | Version | @@ -92,4 +92,4 @@ No modules. | [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | | [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | | [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | - + diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 6810d0fa0c..7cfbf27499 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -80,7 +80,7 @@ module "karpenter" { } ``` - + ## Requirements | Name | Version | @@ -158,7 +158,7 @@ No modules. | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add the the IAM role | `map(any)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the name of the IAM role (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no | -| [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` |
[
"karpenter:karpenter"
]
| no | +| [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` |
[
"karpenter:karpenter"
]
| no | | [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | | [namespace](#input\_namespace) | Namespace to associate with the Karpenter Pod Identity | `string` | `"kube-system"` | no | | [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | @@ -200,4 +200,4 @@ No modules. | [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue | | [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue | | [service\_account](#output\_service\_account) | Service Account associated with the Karpenter Pod Identity | - + diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 8422b0c7ef..14c3abe4ab 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -37,7 +37,7 @@ module "self_managed_node_group" { } ``` - + ## Requirements | Name | Version | @@ -93,8 +93,8 @@ module "self_managed_node_group" { | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [capacity\_rebalance](#input\_capacity\_rebalance) | Indicates whether capacity rebalance is enabled | `bool` | `null` | no | | [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | -| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | -| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | @@ -146,7 +146,7 @@ module "self_managed_node_group" { | [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | | [instance\_maintenance\_policy](#input\_instance\_maintenance\_policy) | If this block is configured, add a instance maintenance policy to the specified Auto Scaling group | `any` | `{}` | no | | [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `{}` | no | -| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated | `any` |
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | +| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated | `any` |
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | | [instance\_requirements](#input\_instance\_requirements) | The attribute requirements for the type of instance. If present then `instance_type` cannot be present | `any` | `{}` | no | | [instance\_type](#input\_instance\_type) | The type of the instance to launch | `string` | `""` | no | | [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | @@ -162,7 +162,7 @@ module "self_managed_node_group" { | [maintenance\_options](#input\_maintenance\_options) | The maintenance options for the instance | `any` | `{}` | no | | [max\_instance\_lifetime](#input\_max\_instance\_lifetime) | The maximum amount of time, in seconds, that an instance can be in service, values must be either equal to 0 or between 604800 and 31536000 seconds | `number` | `null` | no | | [max\_size](#input\_max\_size) | The maximum size of the autoscaling group | `number` | `3` | no | -| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | +| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | | [metrics\_granularity](#input\_metrics\_granularity) | The granularity to associate with the metrics to collect. The only valid value is `1Minute` | `string` | `null` | no | | [min\_elb\_capacity](#input\_min\_elb\_capacity) | Setting this causes Terraform to wait for this number of instances to show up healthy in the ELB only on creation. Updates will not wait on ELB instance number changes | `number` | `null` | no | | [min\_size](#input\_min\_size) | The minimum size of the autoscaling group | `number` | `0` | no | @@ -182,7 +182,7 @@ module "self_managed_node_group" { | [service\_linked\_role\_arn](#input\_service\_linked\_role\_arn) | The ARN of the service-linked role that the ASG will use to call other AWS services | `string` | `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones` | `list(string)` | `null` | no | | [suspended\_processes](#input\_suspended\_processes) | A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly | `list(string)` | `[]` | no | -| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | +| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [target\_group\_arns](#input\_target\_group\_arns) | A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing | `list(string)` | `[]` | no | | [termination\_policies](#input\_termination\_policies) | A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default` | `list(string)` | `[]` | no | @@ -225,4 +225,4 @@ module "self_managed_node_group" { | [launch\_template\_name](#output\_launch\_template\_name) | The name of the launch template | | [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | | [user\_data](#output\_user\_data) | Base64 encoded user data | - + diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index f6f753897b..5478469e94 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -12,7 +12,7 @@ $ terraform apply --auto-approve Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - + ## Requirements | Name | Version | @@ -94,4 +94,4 @@ No inputs. | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | | [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | - + diff --git a/tests/fargate-profile/README.md b/tests/fargate-profile/README.md index f0ac3d5390..16bd7e9fc8 100644 --- a/tests/fargate-profile/README.md +++ b/tests/fargate-profile/README.md @@ -12,7 +12,7 @@ $ terraform apply --auto-approve Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - + ## Requirements | Name | Version | @@ -85,4 +85,4 @@ No inputs. | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | | [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | - + diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 6f5ddce892..6523f4f402 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -12,7 +12,7 @@ $ terraform apply --auto-approve Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - + ## Requirements | Name | Version | @@ -90,4 +90,4 @@ No inputs. | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | | [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | - + diff --git a/tests/user-data/README.md b/tests/user-data/README.md index 3278c7f0f9..88988ab2f4 100644 --- a/tests/user-data/README.md +++ b/tests/user-data/README.md @@ -12,7 +12,7 @@ $ terraform plan $ terraform apply --auto-approve ``` - + ## Requirements | Name | Version | @@ -104,4 +104,4 @@ No inputs. ## Outputs No outputs. - + From 7f219a678ee5712d4d7bed8881749e497130d549 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 21 Sep 2024 14:46:18 +0000 Subject: [PATCH 823/962] chore(release): version 20.24.2 [skip ci] ## [20.24.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.1...v20.24.2) (2024-09-21) ### Bug Fixes * Remove deprecated `inline_policy` from cluster role ([#3163](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3163)) ([8b90872](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8b90872983b9c349ff2e0a71678d687dc32ed626)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfeb4d8e90..9f2559199b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.24.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.1...v20.24.2) (2024-09-21) + + +### Bug Fixes + +* Remove deprecated `inline_policy` from cluster role ([#3163](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3163)) ([8b90872](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8b90872983b9c349ff2e0a71678d687dc32ed626)) + ## [20.24.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.0...v20.24.1) (2024-09-16) From fef655585b33d717c1665bf8151f0573a17dedc2 Mon Sep 17 00:00:00 2001 From: Jami Karvanen Date: Thu, 3 Oct 2024 16:54:52 +0300 Subject: [PATCH 824/962] fix: Add `primary_ipv6` parameter to self-managed-node-group (#3169) Add primary_ipv6 parameter to self-managed-node-group --- modules/self-managed-node-group/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 42411c484e..76201ccd0e 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -417,6 +417,7 @@ resource "aws_launch_template" "this" { ipv6_prefixes = try(network_interfaces.value.ipv6_prefixes, []) network_card_index = try(network_interfaces.value.network_card_index, null) network_interface_id = try(network_interfaces.value.network_interface_id, null) + primary_ipv6 = try(network_interfaces.value.primary_ipv6, null) private_ip_address = try(network_interfaces.value.private_ip_address, null) # Ref: https://github.com/hashicorp/terraform-provider-aws/issues/4570 security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) From d2c671aca25165cc060e2459f5cafb49868a7be3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 3 Oct 2024 13:55:23 +0000 Subject: [PATCH 825/962] chore(release): version 20.24.3 [skip ci] ## [20.24.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.2...v20.24.3) (2024-10-03) ### Bug Fixes * Add `primary_ipv6` parameter to self-managed-node-group ([#3169](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3169)) ([fef6555](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/fef655585b33d717c1665bf8151f0573a17dedc2)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f2559199b..c5787e7944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.24.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.2...v20.24.3) (2024-10-03) + + +### Bug Fixes + +* Add `primary_ipv6` parameter to self-managed-node-group ([#3169](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3169)) ([fef6555](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/fef655585b33d717c1665bf8151f0573a17dedc2)) + ## [20.24.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.1...v20.24.2) (2024-09-21) From 670aa8a79d48e13e70726b0809ac3add3914b58e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 8 Oct 2024 14:15:15 +0000 Subject: [PATCH 826/962] docs: Correct reference module implementation names to be unique in v20 upgrade guide (#3174) --- .pre-commit-config.yaml | 2 +- docs/UPGRADE-20.0.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55b241248d..48e324bc8d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: - '--args=--only=terraform_unused_required_providers' - id: terraform_validate - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer diff --git a/docs/UPGRADE-20.0.md b/docs/UPGRADE-20.0.md index 6f53e326c2..c93a031ee9 100644 --- a/docs/UPGRADE-20.0.md +++ b/docs/UPGRADE-20.0.md @@ -170,7 +170,7 @@ To give users advanced notice and provide some future direction for this module, - ] } -+ module "eks" { ++ module "eks_aws_auth" { + source = "terraform-aws-modules/eks/aws//modules/aws-auth" + version = "~> 20.0" @@ -197,7 +197,7 @@ To give users advanced notice and provide some future direction for this module, ### Karpenter Diff of Before (v19.21) vs After (v20.0) ```diff - module "eks" { + module "eks_karpenter" { source = "terraform-aws-modules/eks/aws//modules/karpenter" - version = "~> 19.21" + version = "~> 20.0" From eb78240617993845a2a85056655b16302ea9a02c Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 11 Oct 2024 16:03:02 +0000 Subject: [PATCH 827/962] fix: Update CI workflow versions to latest (#3176) --- .github/workflows/pr-title.yml | 2 +- .github/workflows/pre-commit.yml | 10 +++++----- .pre-commit-config.yaml | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 3973df4438..1e50760ee7 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -14,7 +14,7 @@ jobs: steps: # Please look up the latest version from # https://github.com/amannn/action-semantic-pull-request/releases - - uses: amannn/action-semantic-pull-request@v5.4.0 + - uses: amannn/action-semantic-pull-request@v5.5.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 71878b4351..a19ff831f9 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -45,14 +45,14 @@ jobs: - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.3.0 + uses: clowdhaus/terraform-min-max@v1.3.1 with: directory: ${{ matrix.directory }} - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory != '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.9.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.11.1 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} tflint-version: ${{ env.TFLINT_VERSION }} @@ -61,7 +61,7 @@ jobs: - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory == '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.9.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.11.1 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} tflint-version: ${{ env.TFLINT_VERSION }} @@ -88,10 +88,10 @@ jobs: - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.3.0 + uses: clowdhaus/terraform-min-max@v1.3.1 - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.9.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.11.1 with: terraform-version: ${{ steps.minMax.outputs.maxVersion }} tflint-version: ${{ env.TFLINT_VERSION }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48e324bc8d..4f0721881c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,6 @@ repos: - '--args=--only=terraform_required_providers' - '--args=--only=terraform_standard_module_structure' - '--args=--only=terraform_workspace_remote' - - '--args=--only=terraform_unused_required_providers' - id: terraform_validate - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 From b2a8617794a782107399b26c1ff4503e0ea5ec3a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 12 Oct 2024 16:44:24 +0000 Subject: [PATCH 828/962] feat: Add support for newly released AL2023 accelerated AMI types (#3177) --- README.md | 6 +++--- docs/compute_resources.md | 4 ++-- examples/eks-managed-node-group/eks-al2.tf | 2 +- examples/eks-managed-node-group/eks-al2023.tf | 2 +- examples/eks-managed-node-group/eks-bottlerocket.tf | 2 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 6 +++--- examples/karpenter/main.tf | 4 ++-- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/eks-al2.tf | 2 +- examples/self-managed-node-group/eks-al2023.tf | 2 +- examples/self-managed-node-group/eks-bottlerocket.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- modules/_user_data/main.tf | 2 ++ modules/eks-managed-node-group/README.md | 6 +++--- modules/eks-managed-node-group/main.tf | 2 ++ modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 6 +++--- modules/self-managed-node-group/main.tf | 4 ++++ modules/self-managed-node-group/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/main.tf | 2 +- tests/eks-managed-node-group/versions.tf | 2 +- tests/fargate-profile/README.md | 4 ++-- tests/fargate-profile/main.tf | 2 +- tests/fargate-profile/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/main.tf | 7 +++++-- tests/self-managed-node-group/versions.tf | 2 +- versions.tf | 2 +- 34 files changed, 58 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 37008e7824..7d3ae81479 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ module "eks" { version = "~> 20.0" cluster_name = "my-cluster" - cluster_version = "1.30" + cluster_version = "1.31" cluster_endpoint_public_access = true @@ -175,7 +175,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.61 | +| [aws](#requirement\_aws) | >= 5.70 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -183,7 +183,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.61 | +| [aws](#provider\_aws) | >= 5.70 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 721c29c1ee..1b033d2814 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -113,7 +113,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 1. The `self-managed-node-group` uses the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version by default: ```hcl - cluster_version = "1.27" + cluster_version = "1.31" # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.27 self_managed_node_groups = { @@ -124,7 +124,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 2. To use Bottlerocket, specify the `ami_type` as one of the respective `"BOTTLEROCKET_*" types` and supply a Bottlerocket OS AMI: ```hcl - cluster_version = "1.27" + cluster_version = "1.31" self_managed_node_groups = { bottlerocket = { diff --git a/examples/eks-managed-node-group/eks-al2.tf b/examples/eks-managed-node-group/eks-al2.tf index 2dfb5b019f..910fa216b2 100644 --- a/examples/eks-managed-node-group/eks-al2.tf +++ b/examples/eks-managed-node-group/eks-al2.tf @@ -3,7 +3,7 @@ module "eks_al2" { version = "~> 20.0" cluster_name = "${local.name}-al2" - cluster_version = "1.30" + cluster_version = "1.31" # EKS Addons cluster_addons = { diff --git a/examples/eks-managed-node-group/eks-al2023.tf b/examples/eks-managed-node-group/eks-al2023.tf index 1b112d23a2..349da821f9 100644 --- a/examples/eks-managed-node-group/eks-al2023.tf +++ b/examples/eks-managed-node-group/eks-al2023.tf @@ -3,7 +3,7 @@ module "eks_al2023" { version = "~> 20.0" cluster_name = "${local.name}-al2023" - cluster_version = "1.30" + cluster_version = "1.31" # EKS Addons cluster_addons = { diff --git a/examples/eks-managed-node-group/eks-bottlerocket.tf b/examples/eks-managed-node-group/eks-bottlerocket.tf index 44efa593c0..01a6878814 100644 --- a/examples/eks-managed-node-group/eks-bottlerocket.tf +++ b/examples/eks-managed-node-group/eks-bottlerocket.tf @@ -3,7 +3,7 @@ module "eks_bottlerocket" { version = "~> 20.0" cluster_name = "${local.name}-bottlerocket" - cluster_version = "1.30" + cluster_version = "1.31" # EKS Addons cluster_addons = { diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index 5bfe6da389..fe57b7a76f 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 3ec2d6c2e7..43d911a8b6 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -89,7 +89,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.61 | +| [aws](#requirement\_aws) | >= 5.70 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -97,8 +97,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.61 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.61 | +| [aws](#provider\_aws) | >= 5.70 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.70 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 72ceff275a..49321c977e 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -62,7 +62,7 @@ module "eks" { source = "../.." cluster_name = local.name - cluster_version = "1.30" + cluster_version = "1.31" # Gives Terraform identity admin access to cluster which will # allow deploying resources (Karpenter) into the cluster @@ -157,7 +157,7 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "1.0.0" + version = "1.0.6" wait = false values = [ diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 8a0624ba0d..b5a6a82f39 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/eks-al2.tf b/examples/self-managed-node-group/eks-al2.tf index be5c65ade5..e015982733 100644 --- a/examples/self-managed-node-group/eks-al2.tf +++ b/examples/self-managed-node-group/eks-al2.tf @@ -3,7 +3,7 @@ module "eks_al2" { version = "~> 20.0" cluster_name = "${local.name}-al2" - cluster_version = "1.30" + cluster_version = "1.31" # EKS Addons cluster_addons = { diff --git a/examples/self-managed-node-group/eks-al2023.tf b/examples/self-managed-node-group/eks-al2023.tf index 701560527f..4d7c5dc7e1 100644 --- a/examples/self-managed-node-group/eks-al2023.tf +++ b/examples/self-managed-node-group/eks-al2023.tf @@ -3,7 +3,7 @@ module "eks_al2023" { version = "~> 20.0" cluster_name = "${local.name}-al2023" - cluster_version = "1.30" + cluster_version = "1.31" # EKS Addons cluster_addons = { diff --git a/examples/self-managed-node-group/eks-bottlerocket.tf b/examples/self-managed-node-group/eks-bottlerocket.tf index 2afb079472..ff2fb2cf3d 100644 --- a/examples/self-managed-node-group/eks-bottlerocket.tf +++ b/examples/self-managed-node-group/eks-bottlerocket.tf @@ -3,7 +3,7 @@ module "eks_bottlerocket" { version = "~> 20.0" cluster_name = "${local.name}-bottlerocket" - cluster_version = "1.30" + cluster_version = "1.31" # EKS Addons cluster_addons = { diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index 5bfe6da389..fe57b7a76f 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } } } diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index 79b8cbae5d..e66cd3d466 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -30,6 +30,8 @@ locals { WINDOWS_FULL_2022_x86_64 = "windows" AL2023_x86_64_STANDARD = "al2023" AL2023_ARM_64_STANDARD = "al2023" + AL2023_x86_64_NEURON = "al2023" + AL2023_x86_64_NVIDIA = "al2023" } # Try to use `ami_type` first, but fall back to current, default behavior # TODO - will be removed in v21.0 diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index de0c95b103..c64f006fc9 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -10,7 +10,7 @@ module "eks_managed_node_group" { name = "separate-eks-mng" cluster_name = "my-cluster" - cluster_version = "1.27" + cluster_version = "1.31" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.61 | +| [aws](#requirement\_aws) | >= 5.70 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.61 | +| [aws](#provider\_aws) | >= 5.70 | ## Modules diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 9c0c78c9c4..42c221dc91 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -362,6 +362,8 @@ locals { WINDOWS_FULL_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-${local.ssm_cluster_version}" AL2023_x86_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/standard/recommended/release_version" AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/standard/recommended/release_version" + AL2023_x86_64_NEURON = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/neuron/recommended/release_version" + AL2023_x86_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/nvidia/recommended/release_version" } # The Windows SSM params currently do not have a release version, so we have to get the full output JSON blob and parse out the release version diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 5bfe6da389..fe57b7a76f 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index e6be895ebe..a030f5ae07 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.61 | +| [aws](#requirement\_aws) | >= 5.70 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.61 | +| [aws](#provider\_aws) | >= 5.70 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 5bfe6da389..fe57b7a76f 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 7cfbf27499..b2bcc1fc6f 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.61 | +| [aws](#requirement\_aws) | >= 5.70 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.61 | +| [aws](#provider\_aws) | >= 5.70 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 5bfe6da389..fe57b7a76f 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 14c3abe4ab..81d58579dd 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -10,7 +10,7 @@ module "self_managed_node_group" { name = "separate-self-mng" cluster_name = "my-cluster" - cluster_version = "1.27" + cluster_version = "1.31" cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.61 | +| [aws](#requirement\_aws) | >= 5.70 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.61 | +| [aws](#provider\_aws) | >= 5.70 | ## Modules diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 76201ccd0e..77ca1856c8 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -24,6 +24,8 @@ locals { WINDOWS_FULL_2022_x86_64 = "windows" AL2023_x86_64_STANDARD = "al2023" AL2023_ARM_64_STANDARD = "al2023" + AL2023_x86_64_NEURON = "al2023" + AL2023_x86_64_NVIDIA = "al2023" } user_data_type = local.ami_type_to_user_data_type[var.ami_type] @@ -43,6 +45,8 @@ locals { WINDOWS_FULL_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-${local.ssm_cluster_version}/image_id" AL2023_x86_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/standard/recommended/image_id" AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/standard/recommended/image_id" + AL2023_x86_64_NEURON = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/neuron/recommended/image_id" + AL2023_x86_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/nvidia/recommended/image_id" } } diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 5bfe6da389..fe57b7a76f 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } } } diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 5478469e94..70c3b10746 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.61 | +| [aws](#requirement\_aws) | >= 5.70 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.61 | +| [aws](#provider\_aws) | >= 5.70 | ## Modules diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index dfe7aea768..5cb5ccf8b5 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -7,7 +7,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.30" + cluster_version = "1.31" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index 5bfe6da389..fe57b7a76f 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } } } diff --git a/tests/fargate-profile/README.md b/tests/fargate-profile/README.md index 16bd7e9fc8..34872e8b1a 100644 --- a/tests/fargate-profile/README.md +++ b/tests/fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.61 | +| [aws](#requirement\_aws) | >= 5.70 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.61 | +| [aws](#provider\_aws) | >= 5.70 | ## Modules diff --git a/tests/fargate-profile/main.tf b/tests/fargate-profile/main.tf index 6106022373..fc9afc76ed 100644 --- a/tests/fargate-profile/main.tf +++ b/tests/fargate-profile/main.tf @@ -6,7 +6,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${basename(path.cwd)}" - cluster_version = "1.30" + cluster_version = "1.31" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" diff --git a/tests/fargate-profile/versions.tf b/tests/fargate-profile/versions.tf index 5bfe6da389..fe57b7a76f 100644 --- a/tests/fargate-profile/versions.tf +++ b/tests/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 6523f4f402..1dbf43bac4 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.61 | +| [aws](#requirement\_aws) | >= 5.70 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.61 | +| [aws](#provider\_aws) | >= 5.70 | ## Modules diff --git a/tests/self-managed-node-group/main.tf b/tests/self-managed-node-group/main.tf index 7fc6171e95..b2c1423445 100644 --- a/tests/self-managed-node-group/main.tf +++ b/tests/self-managed-node-group/main.tf @@ -7,7 +7,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.29" + cluster_version = "1.31" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -138,6 +138,7 @@ module "eks" { max_size = 5 desired_size = 2 + ami_type = "AL2_x86_64" bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" use_mixed_instances_policy = true @@ -172,7 +173,8 @@ module "eks" { max_size = 7 desired_size = 1 - ami_id = data.aws_ami.eks_default.id + ami_id = data.aws_ami.eks_default.id + ami_type = "AL2_x86_64" pre_bootstrap_user_data = <<-EOT export FOO=bar @@ -213,6 +215,7 @@ module "eks" { max_size = 2 desired_size = 1 + ami_type = "AL2_x86_64" bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" instance_type = null diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index 5bfe6da389..fe57b7a76f 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } } } diff --git a/versions.tf b/versions.tf index 00d8754704..4717288ded 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.61" + version = ">= 5.70" } tls = { source = "hashicorp/tls" From 159db95b6d99f7f8a93a403a7aeb7ded079345cd Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 12 Oct 2024 16:44:50 +0000 Subject: [PATCH 829/962] chore(release): version 20.25.0 [skip ci] ## [20.25.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.3...v20.25.0) (2024-10-12) ### Features * Add support for newly released AL2023 accelerated AMI types ([#3177](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3177)) ([b2a8617](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b2a8617794a782107399b26c1ff4503e0ea5ec3a)) ### Bug Fixes * Update CI workflow versions to latest ([#3176](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3176)) ([eb78240](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/eb78240617993845a2a85056655b16302ea9a02c)) --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5787e7944..5a205edb3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. +## [20.25.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.3...v20.25.0) (2024-10-12) + + +### Features + +* Add support for newly released AL2023 accelerated AMI types ([#3177](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3177)) ([b2a8617](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b2a8617794a782107399b26c1ff4503e0ea5ec3a)) + + +### Bug Fixes + +* Update CI workflow versions to latest ([#3176](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3176)) ([eb78240](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/eb78240617993845a2a85056655b16302ea9a02c)) + ## [20.24.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.2...v20.24.3) (2024-10-03) From 6974a5e1582a4ed2d8b1f9a07cdacd156ba5ffef Mon Sep 17 00:00:00 2001 From: dtscssap <146767262+dtscssap@users.noreply.github.com> Date: Sat, 12 Oct 2024 12:57:19 -0400 Subject: [PATCH 830/962] feat: Add support for `desired_capacity_type` (named `desired_size_type`) on self-managed node group (#3166) * set desired_capacity_type * fix: Add variable with `null` type and pass through node group --------- Co-authored-by: Bryant Biggs --- modules/self-managed-node-group/README.md | 1 + modules/self-managed-node-group/main.tf | 1 + modules/self-managed-node-group/variables.tf | 6 ++++++ node_groups.tf | 1 + 4 files changed, 9 insertions(+) diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 81d58579dd..7b9ffc5a8b 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -117,6 +117,7 @@ module "self_managed_node_group" { | [default\_instance\_warmup](#input\_default\_instance\_warmup) | Amount of time, in seconds, until a newly launched instance can contribute to the Amazon CloudWatch metrics. This delay lets an instance finish initializing before Amazon EC2 Auto Scaling aggregates instance metrics, resulting in more reliable usage data | `number` | `null` | no | | [delete\_timeout](#input\_delete\_timeout) | Delete timeout to wait for destroying autoscaling group | `string` | `null` | no | | [desired\_size](#input\_desired\_size) | The number of Amazon EC2 instances that should be running in the autoscaling group | `number` | `1` | no | +| [desired\_size\_type](#input\_desired\_size\_type) | The unit of measurement for the value specified for `desired_size`. Supported for attribute-based instance type selection only. Valid values: `units`, `vcpu`, `memory-mib` | `string` | `null` | no | | [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance will be EBS-optimized | `bool` | `null` | no | | [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 77ca1856c8..127591081b 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -502,6 +502,7 @@ resource "aws_autoscaling_group" "this" { default_cooldown = var.default_cooldown default_instance_warmup = var.default_instance_warmup desired_capacity = var.desired_size + desired_capacity_type = var.desired_size_type enabled_metrics = var.enabled_metrics force_delete = var.force_delete force_delete_warm_pool = var.force_delete_warm_pool diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 92121ea750..fd2216d300 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -420,6 +420,12 @@ variable "desired_size" { default = 1 } +variable "desired_size_type" { + description = "The unit of measurement for the value specified for `desired_size`. Supported for attribute-based instance type selection only. Valid values: `units`, `vcpu`, `memory-mib`" + type = string + default = null +} + variable "ignore_failed_scaling_activities" { description = "Whether to ignore failed Auto Scaling scaling activities while waiting for capacity." type = bool diff --git a/node_groups.tf b/node_groups.tf index de75fc9133..4ac638100a 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -435,6 +435,7 @@ module "self_managed_node_group" { min_size = try(each.value.min_size, var.self_managed_node_group_defaults.min_size, 0) max_size = try(each.value.max_size, var.self_managed_node_group_defaults.max_size, 3) desired_size = try(each.value.desired_size, var.self_managed_node_group_defaults.desired_size, 1) + desired_size_type = try(each.value.desired_size_type, var.self_managed_node_group_defaults.desired_size_type, null) capacity_rebalance = try(each.value.capacity_rebalance, var.self_managed_node_group_defaults.capacity_rebalance, null) min_elb_capacity = try(each.value.min_elb_capacity, var.self_managed_node_group_defaults.min_elb_capacity, null) wait_for_elb_capacity = try(each.value.wait_for_elb_capacity, var.self_managed_node_group_defaults.wait_for_elb_capacity, null) From 00d4cc1373d97a5abfa05b7cc75e9c9a189e4d5f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 12 Oct 2024 16:57:44 +0000 Subject: [PATCH 831/962] chore(release): version 20.26.0 [skip ci] ## [20.26.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.25.0...v20.26.0) (2024-10-12) ### Features * Add support for `desired_capacity_type` (named `desired_size_type`) on self-managed node group ([#3166](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3166)) ([6974a5e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6974a5e1582a4ed2d8b1f9a07cdacd156ba5ffef)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a205edb3c..195f5c5c46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.26.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.25.0...v20.26.0) (2024-10-12) + + +### Features + +* Add support for `desired_capacity_type` (named `desired_size_type`) on self-managed node group ([#3166](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3166)) ([6974a5e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6974a5e1582a4ed2d8b1f9a07cdacd156ba5ffef)) + ## [20.25.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.24.3...v20.25.0) (2024-10-12) From 4abc779c0e29fdf64d6c725ef0c5f7de8a708c46 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 22 Oct 2024 15:08:16 +0000 Subject: [PATCH 832/962] docs: Correct doc links following `examples`/`tests` re-org (#3187) --- docs/UPGRADE-18.0.md | 2 +- docs/compute_resources.md | 6 +++--- docs/user_data.md | 2 +- modules/_user_data/README.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/UPGRADE-18.0.md b/docs/UPGRADE-18.0.md index 3f25ca8e43..8b3d0accad 100644 --- a/docs/UPGRADE-18.0.md +++ b/docs/UPGRADE-18.0.md @@ -48,7 +48,7 @@ See more information [here](https://github.com/terraform-aws-modules/terraform-a - Additional changes for the `self-managed-node-group` sub-module over the previous `node_groups` variable include: - The underlying autoscaling group and launch template have been updated to more closely match that of the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module and the features it offers - The previous iteration used a count over a list of node group definitions which was prone to disruptive updates; this is now replaced with a map/for_each to align with that of the EKS managed node group and Fargate profile behaviors/style -- The user data configuration supported across the module has been completely revamped. A new `_user_data` internal sub-module has been created to consolidate all user data configuration in one location which provides better support for testability (via the [`examples/user_data`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) example). The new sub-module supports nearly all possible combinations including the ability to allow users to provide their own user data template which will be rendered by the module. See the `examples/user_data` example project for the full plethora of example configuration possibilities and more details on the logic of the design can be found in the [`modules/_user_data`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/_user_data_) directory. +- The user data configuration supported across the module has been completely revamped. A new `_user_data` internal sub-module has been created to consolidate all user data configuration in one location which provides better support for testability (via the [`tests/user-data`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/tests/user-data) example). The new sub-module supports nearly all possible combinations including the ability to allow users to provide their own user data template which will be rendered by the module. See the `tests/user-data` example project for the full plethora of example configuration possibilities and more details on the logic of the design can be found in the [`modules/_user_data`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/_user_data_) directory. - Resource name changes may cause issues with existing resources. For example, security groups and IAM roles cannot be renamed, they must be recreated. Recreation of these resources may also trigger a recreation of the cluster. To use the legacy (< 18.x) resource naming convention, set `prefix_separator` to "". - Security group usage has been overhauled to provide only the bare minimum network connectivity required to launch a bare bones cluster. See the [security group documentation section](https://github.com/terraform-aws-modules/terraform-aws-eks#security-groups) for more details. Users upgrading to v18.x will want to review the rules they have in place today versus the rules provisioned by the v18.x module and ensure to make any necessary adjustments for their specific workload. diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 1b033d2814..6b74c4a507 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -104,7 +104,7 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ } ``` -See the [`examples/eks_managed_node_group/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group) for a working example of various configurations. +See the [`examples/eks-managed-node-group/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-managed-node-group) for a working example of various configurations. ### Self Managed Node Groups @@ -134,11 +134,11 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com } ``` -See the [`examples/self_managed_node_group/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self_managed_node_group) for a working example of various configurations. +See the [`examples/self-managed-node-group/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self-managed-node-group) for a working example of various configurations. ### Fargate Profiles -Fargate profiles are straightforward to use and therefore no further details are provided here. See the [`examples/fargate_profile/` example](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile) for a working example of various configurations. +Fargate profiles are straightforward to use and therefore no further details are provided here. See the [`tests/fargate-profile/` tests](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/tests/fargate-profile) for a working example of various configurations. ### Default Configurations diff --git a/docs/user_data.md b/docs/user_data.md index 1ddc5d1aa7..3ca2263deb 100644 --- a/docs/user_data.md +++ b/docs/user_data.md @@ -1,6 +1,6 @@ # User Data & Bootstrapping -Users can see the various methods of using and providing user data through the [user data examples](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) as well more detailed information on the design and possible configurations via the [user data module itself](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/_user_data) +Users can see the various methods of using and providing user data through the [user data tests](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/tests/user-data) as well more detailed information on the design and possible configurations via the [user data module itself](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/modules/_user_data) ## Summary diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 09c4b98a4e..185335b16a 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -2,7 +2,7 @@ Configuration in this directory renders the appropriate user data for the given inputs. See [`docs/user_data.md`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/user_data.md) for more info. -See [`examples/user_data/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data) for various examples using this module. +See [`tests/user-data/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/tests/user-data) for various tests cases using this module. ## Requirements From dea6c44b459a546b1386563dfd497bc9d766bfe1 Mon Sep 17 00:00:00 2001 From: joey100 Date: Sun, 27 Oct 2024 08:12:25 +0800 Subject: [PATCH 833/962] fix: Use dynamic partition data source to determine DNS suffix for Karpenter EC2 pass role permission (#3193) * fix karpenter iam passrole to ec2 api bug, to support aws cn * fix: Use dyanmic partition value for DNS suffix --------- Co-authored-by: Shuiping Co-authored-by: Bryant Biggs --- modules/karpenter/policy.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/karpenter/policy.tf b/modules/karpenter/policy.tf index 456a27f417..7fb04e47b2 100644 --- a/modules/karpenter/policy.tf +++ b/modules/karpenter/policy.tf @@ -195,7 +195,7 @@ data "aws_iam_policy_document" "v033" { condition { test = "StringEquals" variable = "iam:PassedToService" - values = ["ec2.amazonaws.com"] + values = ["ec2.${local.dns_suffix}"] } } @@ -584,7 +584,7 @@ data "aws_iam_policy_document" "v1" { condition { test = "StringEquals" variable = "iam:PassedToService" - values = ["ec2.amazonaws.com"] + values = ["ec2.${local.dns_suffix}"] } } From 769633253996237a79fe4c4cb49ec0eadfae5e69 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 27 Oct 2024 00:13:02 +0000 Subject: [PATCH 834/962] chore(release): version 20.26.1 [skip ci] ## [20.26.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.26.0...v20.26.1) (2024-10-27) ### Bug Fixes * Use dynamic partition data source to determine DNS suffix for Karpenter EC2 pass role permission ([#3193](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3193)) ([dea6c44](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/dea6c44b459a546b1386563dfd497bc9d766bfe1)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 195f5c5c46..84131a93aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.26.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.26.0...v20.26.1) (2024-10-27) + + +### Bug Fixes + +* Use dynamic partition data source to determine DNS suffix for Karpenter EC2 pass role permission ([#3193](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3193)) ([dea6c44](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/dea6c44b459a546b1386563dfd497bc9d766bfe1)) + ## [20.26.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.25.0...v20.26.0) (2024-10-12) From 1b0ac832647dcf0425aedba119fa8276008cbe28 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 1 Nov 2024 21:50:19 +0000 Subject: [PATCH 835/962] feat: Add support for zonal shift (#3195) --- README.md | 5 +++-- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 6 +++--- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- main.tf | 8 ++++++++ modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/main.tf | 4 ++++ tests/eks-managed-node-group/versions.tf | 2 +- tests/fargate-profile/README.md | 4 ++-- tests/fargate-profile/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- variables.tf | 6 ++++++ versions.tf | 2 +- 23 files changed, 49 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 7d3ae81479..b272a6475d 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.70 | +| [aws](#requirement\_aws) | >= 5.74 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -183,7 +183,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.70 | +| [aws](#provider\_aws) | >= 5.74 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | @@ -268,6 +268,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | | [cluster\_upgrade\_policy](#input\_cluster\_upgrade\_policy) | Configuration block for the cluster upgrade policy | `any` | `{}` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.27`) | `string` | `null` | no | +| [cluster\_zonal\_shift\_config](#input\_cluster\_zonal\_shift\_config) | Configuration block for the cluster zonal shift | `any` | `{}` | no | | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index fe57b7a76f..3cc97fa038 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 43d911a8b6..b621a36591 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -89,7 +89,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.70 | +| [aws](#requirement\_aws) | >= 5.74 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -97,8 +97,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.70 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.70 | +| [aws](#provider\_aws) | >= 5.74 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.74 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index b5a6a82f39..0c0cc6c763 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index fe57b7a76f..3cc97fa038 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } } } diff --git a/main.tf b/main.tf index b891b26d0e..0b62869c24 100644 --- a/main.tf +++ b/main.tf @@ -92,6 +92,14 @@ resource "aws_eks_cluster" "this" { } } + dynamic "zonal_shift_config" { + for_each = length(var.cluster_zonal_shift_config) > 0 ? [var.cluster_zonal_shift_config] : [] + + content { + enabled = try(zonal_shift_config.value.enabled, null) + } + } + tags = merge( { terraform-aws-modules = "eks" }, var.tags, diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index c64f006fc9..709885898d 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.70 | +| [aws](#requirement\_aws) | >= 5.74 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.70 | +| [aws](#provider\_aws) | >= 5.74 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index fe57b7a76f..3cc97fa038 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index a030f5ae07..732cd7b455 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.70 | +| [aws](#requirement\_aws) | >= 5.74 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.70 | +| [aws](#provider\_aws) | >= 5.74 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index fe57b7a76f..3cc97fa038 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index b2bcc1fc6f..2ca911d156 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.70 | +| [aws](#requirement\_aws) | >= 5.74 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.70 | +| [aws](#provider\_aws) | >= 5.74 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index fe57b7a76f..3cc97fa038 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 7b9ffc5a8b..c8961a104b 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.70 | +| [aws](#requirement\_aws) | >= 5.74 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.70 | +| [aws](#provider\_aws) | >= 5.74 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index fe57b7a76f..3cc97fa038 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } } } diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 70c3b10746..39af8f08ed 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.70 | +| [aws](#requirement\_aws) | >= 5.74 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.70 | +| [aws](#provider\_aws) | >= 5.74 | ## Modules diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index 5cb5ccf8b5..9c59f30507 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -65,6 +65,10 @@ module "eks" { support_type = "STANDARD" } + cluster_zonal_shift_config = { + enabled = true + } + vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index fe57b7a76f..3cc97fa038 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } } } diff --git a/tests/fargate-profile/README.md b/tests/fargate-profile/README.md index 34872e8b1a..a7af7cd080 100644 --- a/tests/fargate-profile/README.md +++ b/tests/fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.70 | +| [aws](#requirement\_aws) | >= 5.74 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.70 | +| [aws](#provider\_aws) | >= 5.74 | ## Modules diff --git a/tests/fargate-profile/versions.tf b/tests/fargate-profile/versions.tf index fe57b7a76f..3cc97fa038 100644 --- a/tests/fargate-profile/versions.tf +++ b/tests/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 1dbf43bac4..0691b7aa0b 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.70 | +| [aws](#requirement\_aws) | >= 5.74 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.70 | +| [aws](#provider\_aws) | >= 5.74 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index fe57b7a76f..3cc97fa038 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } } } diff --git a/variables.tf b/variables.tf index 420248c744..7a7226b96a 100644 --- a/variables.tf +++ b/variables.tf @@ -50,6 +50,12 @@ variable "cluster_upgrade_policy" { default = {} } +variable "cluster_zonal_shift_config" { + description = "Configuration block for the cluster zonal shift" + type = any + default = {} +} + variable "cluster_additional_security_group_ids" { description = "List of additional, externally created security group IDs to attach to the cluster control plane" type = list(string) diff --git a/versions.tf b/versions.tf index 4717288ded..090ca7b00b 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.70" + version = ">= 5.74" } tls = { source = "hashicorp/tls" From ac14562541844ffd1646a84e90b68b15fab26ecd Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 1 Nov 2024 21:50:50 +0000 Subject: [PATCH 836/962] chore(release): version 20.27.0 [skip ci] ## [20.27.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.26.1...v20.27.0) (2024-11-01) ### Features * Add support for zonal shift ([#3195](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3195)) ([1b0ac83](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/1b0ac832647dcf0425aedba119fa8276008cbe28)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84131a93aa..6169e0c148 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.27.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.26.1...v20.27.0) (2024-11-01) + + +### Features + +* Add support for zonal shift ([#3195](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3195)) ([1b0ac83](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/1b0ac832647dcf0425aedba119fa8276008cbe28)) + ## [20.26.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.26.0...v20.26.1) (2024-10-27) From c6da22c78f60a8643a6c76f97c93724f4e1f4e5a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 2 Nov 2024 13:33:13 +0000 Subject: [PATCH 837/962] feat: Add support for creating `efa-only` network interfaces (#3196) --- .pre-commit-config.yaml | 2 +- modules/eks-managed-node-group/README.md | 2 + modules/eks-managed-node-group/main.tf | 3 +- modules/eks-managed-node-group/variables.tf | 13 ++ modules/self-managed-node-group/README.md | 2 + modules/self-managed-node-group/main.tf | 5 +- modules/self-managed-node-group/variables.tf | 13 ++ node_groups.tf | 4 + tests/eks-managed-node-group/main.tf | 101 +++++++++----- tests/self-managed-node-group/main.tf | 130 ++++++++++++------- 10 files changed, 196 insertions(+), 79 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f0721881c..baacea01bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.96.1 + rev: v1.96.2 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 709885898d..ace9106c3d 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -132,9 +132,11 @@ module "eks_managed_node_group" { | [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | | [disk\_size](#input\_disk\_size) | Disk size in GiB for nodes. Defaults to `20`. Only valid when `use_custom_launch_template` = `false` | `number` | `null` | no | | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance(s) will be EBS-optimized | `bool` | `null` | no | +| [efa\_indices](#input\_efa\_indices) | The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true` | `list(number)` |
[
0
]
| no | | [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | | [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | | [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template. Only valid when using a custom AMI via `ami_id` | `bool` | `false` | no | +| [enable\_efa\_only](#input\_enable\_efa\_only) | Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later | `bool` | `false` | no | | [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | | [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 42c221dc91..c1581439d2 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -44,13 +44,14 @@ locals { efa_instance_type = try(element(var.instance_types, 0), "") num_network_cards = try(data.aws_ec2_instance_type.this[0].maximum_network_cards, 0) + # Primary network interface must be EFA, remaining can be EFA or EFA-only efa_network_interfaces = [ for i in range(local.num_network_cards) : { associate_public_ip_address = false delete_on_termination = true device_index = i == 0 ? 0 : 1 network_card_index = i - interface_type = "efa" + interface_type = var.enable_efa_only ? contains(concat([0], var.efa_indices), i) ? "efa" : "efa-only" : "efa" } ] diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index bb60b85665..e0ee435785 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -285,6 +285,19 @@ variable "enable_efa_support" { default = false } +# TODO - make this true by default at next breaking change (remove variable, only pass indices) +variable "enable_efa_only" { + description = "Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later" + type = bool + default = false +} + +variable "efa_indices" { + description = "The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true`" + type = list(number) + default = [0] +} + variable "network_interfaces" { description = "Customize network interfaces to be attached at instance boot time" type = list(any) diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index c8961a104b..d2c53be59a 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -120,8 +120,10 @@ module "self_managed_node_group" { | [desired\_size\_type](#input\_desired\_size\_type) | The unit of measurement for the value specified for `desired_size`. Supported for attribute-based instance type selection only. Valid values: `units`, `vcpu`, `memory-mib` | `string` | `null` | no | | [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance will be EBS-optimized | `bool` | `null` | no | +| [efa\_indices](#input\_efa\_indices) | The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true` | `list(number)` |
[
0
]
| no | | [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | | [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | +| [enable\_efa\_only](#input\_enable\_efa\_only) | Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later | `bool` | `false` | no | | [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | | [enabled\_metrics](#input\_enabled\_metrics) | A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances` | `list(string)` | `[]` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 127591081b..484a92e340 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -90,7 +90,7 @@ module "user_data" { ################################################################################ data "aws_ec2_instance_type" "this" { - count = local.enable_efa_support ? 1 : 0 + count = var.create && var.enable_efa_support ? 1 : 0 instance_type = var.instance_type } @@ -101,13 +101,14 @@ locals { instance_type_provided = var.instance_type != "" num_network_cards = try(data.aws_ec2_instance_type.this[0].maximum_network_cards, 0) + # Primary network interface must be EFA, remaining can be EFA or EFA-only efa_network_interfaces = [ for i in range(local.num_network_cards) : { associate_public_ip_address = false delete_on_termination = true device_index = i == 0 ? 0 : 1 network_card_index = i - interface_type = "efa" + interface_type = var.enable_efa_only ? contains(concat([0], var.efa_indices), i) ? "efa" : "efa-only" : "efa" } ] diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index fd2216d300..9076dab5af 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -334,6 +334,19 @@ variable "enable_efa_support" { default = false } +# TODO - make this true by default at next breaking change (remove variable, only pass indices) +variable "enable_efa_only" { + description = "Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later" + type = bool + default = false +} + +variable "efa_indices" { + description = "The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true`" + type = list(number) + default = [0] +} + variable "metadata_options" { description = "Customize the metadata options for the instance" type = map(string) diff --git a/node_groups.tf b/node_groups.tf index 4ac638100a..a8e499abfa 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -375,6 +375,8 @@ module "eks_managed_node_group" { metadata_options = try(each.value.metadata_options, var.eks_managed_node_group_defaults.metadata_options, local.metadata_options) enable_monitoring = try(each.value.enable_monitoring, var.eks_managed_node_group_defaults.enable_monitoring, true) enable_efa_support = try(each.value.enable_efa_support, var.eks_managed_node_group_defaults.enable_efa_support, false) + enable_efa_only = try(each.value.enable_efa_only, var.eks_managed_node_group_defaults.enable_efa_only, false) + efa_indices = try(each.value.efa_indices, var.eks_managed_node_group_defaults.efa_indices, [0]) create_placement_group = try(each.value.create_placement_group, var.eks_managed_node_group_defaults.create_placement_group, false) placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, {}) placement_group_az = try(each.value.placement_group_az, var.eks_managed_node_group_defaults.placement_group_az, null) @@ -526,6 +528,8 @@ module "self_managed_node_group" { metadata_options = try(each.value.metadata_options, var.self_managed_node_group_defaults.metadata_options, local.metadata_options) enable_monitoring = try(each.value.enable_monitoring, var.self_managed_node_group_defaults.enable_monitoring, true) enable_efa_support = try(each.value.enable_efa_support, var.self_managed_node_group_defaults.enable_efa_support, false) + enable_efa_only = try(each.value.enable_efa_only, var.self_managed_node_group_defaults.enable_efa_only, false) + efa_indices = try(each.value.efa_indices, var.self_managed_node_group_defaults.efa_indices, [0]) network_interfaces = try(each.value.network_interfaces, var.self_managed_node_group_defaults.network_interfaces, []) placement = try(each.value.placement, var.self_managed_node_group_defaults.placement, {}) maintenance_options = try(each.value.maintenance_options, var.self_managed_node_group_defaults.maintenance_options, {}) diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index 9c59f30507..7292e765dc 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -74,7 +74,7 @@ module "eks" { control_plane_subnet_ids = module.vpc.intra_subnets eks_managed_node_group_defaults = { - ami_type = "AL2_x86_64" + ami_type = "AL2023_x86_64_STANDARD" instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] } @@ -184,7 +184,7 @@ module "eks" { # Use a custom AMI custom_ami = { - ami_type = "AL2_ARM_64" + ami_type = "AL2023_ARM_64_STANDARD" # Current default AMI used by managed node groups - pseudo "custom" ami_id = data.aws_ami.eks_default_arm.image_id @@ -211,13 +211,28 @@ module "eks" { ami_id = data.aws_ami.eks_default.image_id enable_bootstrap_user_data = true - pre_bootstrap_user_data = <<-EOT - export FOO=bar - EOT - - post_bootstrap_user_data = <<-EOT - echo "you are free little kubelet!" - EOT + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] + + # This is only possible with a custom AMI or self-managed node group + cloudinit_post_nodeadm = [{ + content = <<-EOT + echo "All done" + EOT + content_type = "text/x-shellscript; charset=\"us-ascii\"" + }] capacity_type = "SPOT" force_update_version = true @@ -227,14 +242,6 @@ module "eks" { GithubOrg = "terraform-aws-modules" } - taints = [ - { - key = "dedicated" - value = "gpuGroup" - effect = "NO_SCHEDULE" - } - ] - update_config = { max_unavailable_percentage = 33 # or set `max_unavailable` } @@ -306,19 +313,53 @@ module "eks" { # Can be enabled when appropriate for testing/validation create = false - ami_type = "AL2_x86_64_GPU" - instance_types = ["trn1n.32xlarge"] + # The EKS AL2023 NVIDIA AMI provides all of the necessary components + # for accelerated workloads w/ EFA + ami_type = "AL2023_x86_64_NVIDIA" + instance_types = ["p5e.48xlarge"] - enable_efa_support = true - pre_bootstrap_user_data = <<-EOT - # Mount NVME instance store volumes since they are typically - # available on instances that support EFA - setup-local-disks raid0 - EOT + # Mount instance store volumes in RAID-0 for kubelet and containerd + # https://github.com/awslabs/amazon-eks-ami/blob/master/doc/USER_GUIDE.md#raid-0-for-kubelet-and-containerd-raid0 + cloudinit_pre_nodeadm = [ + { + content_type = "application/node.eks.aws" + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + instance: + localStorage: + strategy: RAID0 + EOT + } + ] + + # This will: + # 1. Create a placement group to place the instances close to one another + # 2. Ignore subnets that reside in AZs that do not support the instance type + # 3. Expose all of the available EFA interfaces on the launch template + enable_efa_support = true + enable_efa_only = true + efa_indices = [0, 4, 8, 12] - min_size = 2 - max_size = 2 - desired_size = 2 + min_size = 1 + max_size = 1 + desired_size = 1 + + labels = { + "vpc.amazonaws.com/efa.present" = "true" + "nvidia.com/gpu.present" = "true" + } + + taints = { + # Ensure only GPU workloads are scheduled on this node group + gpu = { + key = "nvidia.com/gpu" + value = "true" + effect = "NO_SCHEDULE" + } + } } } @@ -532,7 +573,7 @@ data "aws_ami" "eks_default" { filter { name = "name" - values = ["amazon-eks-node-${local.cluster_version}-v*"] + values = ["amazon-eks-node-al2023-x86_64-standard-${local.cluster_version}-v*"] } } @@ -542,7 +583,7 @@ data "aws_ami" "eks_default_arm" { filter { name = "name" - values = ["amazon-eks-arm64-node-${local.cluster_version}-v*"] + values = ["amazon-eks-node-al2023-arm64-standard-${local.cluster_version}-v*"] } } diff --git a/tests/self-managed-node-group/main.tf b/tests/self-managed-node-group/main.tf index b2c1423445..dee3274dc4 100644 --- a/tests/self-managed-node-group/main.tf +++ b/tests/self-managed-node-group/main.tf @@ -61,6 +61,9 @@ module "eks" { } self_managed_node_group_defaults = { + ami_type = "AL2023_x86_64_STANDARD" + ami_id = data.aws_ami.eks_default.image_id + # enable discovery of autoscaling groups by cluster-autoscaler autoscaling_group_tags = { "k8s.io/cluster-autoscaler/enabled" : true, @@ -72,29 +75,6 @@ module "eks" { # Default node group - as provisioned by the module defaults default_node_group = {} - # AL2023 node group utilizing new user data format which utilizes nodeadm - # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh) - al2023_nodeadm = { - ami_type = "AL2023_x86_64_STANDARD" - - cloudinit_pre_nodeadm = [ - { - content_type = "application/node.eks.aws" - content = <<-EOT - --- - apiVersion: node.eks.aws/v1alpha1 - kind: NodeConfig - spec: - kubelet: - config: - shutdownGracePeriod: 30s - featureGates: - DisableKubeletCloudCredentialProviders: true - EOT - } - ] - } - # Bottlerocket node group bottlerocket = { name = "bottlerocket-self-mng" @@ -138,8 +118,18 @@ module "eks" { max_size = 5 desired_size = 2 - ami_type = "AL2_x86_64" - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + flags: + - --node-labels=node.kubernetes.io/lifecycle=spot + EOT + content_type = "application/node.eks.aws" + }] use_mixed_instances_policy = true mixed_instances_policy = { @@ -173,16 +163,18 @@ module "eks" { max_size = 7 desired_size = 1 - ami_id = data.aws_ami.eks_default.id - ami_type = "AL2_x86_64" - - pre_bootstrap_user_data = <<-EOT - export FOO=bar - EOT - - post_bootstrap_user_data = <<-EOT - echo "you are free little kubelet!" - EOT + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + flags: + - --node-labels=node.kubernetes.io/lifecycle=spot + EOT + content_type = "application/node.eks.aws" + }] instance_type = "m6i.large" @@ -215,9 +207,23 @@ module "eks" { max_size = 2 desired_size = 1 - ami_type = "AL2_x86_64" bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] + instance_type = null # launch template configuration @@ -290,19 +296,53 @@ module "eks" { # Can be enabled when appropriate for testing/validation create = false - ami_type = "AL2_x86_64_GPU" - instance_type = "trn1n.32xlarge" + # The EKS AL2023 NVIDIA AMI provides all of the necessary components + # for accelerated workloads w/ EFA + ami_type = "AL2023_x86_64_NVIDIA" + instance_types = ["p5e.48xlarge"] - enable_efa_support = true - pre_bootstrap_user_data = <<-EOT - # Mount NVME instance store volumes since they are typically - # available on instances that support EFA - setup-local-disks raid0 - EOT + # Mount instance store volumes in RAID-0 for kubelet and containerd + # https://github.com/awslabs/amazon-eks-ami/blob/master/doc/USER_GUIDE.md#raid-0-for-kubelet-and-containerd-raid0 + cloudinit_pre_nodeadm = [ + { + content_type = "application/node.eks.aws" + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + instance: + localStorage: + strategy: RAID0 + EOT + } + ] + + # This will: + # 1. Create a placement group to place the instances close to one another + # 2. Ignore subnets that reside in AZs that do not support the instance type + # 3. Expose all of the available EFA interfaces on the launch template + enable_efa_support = true + enable_efa_only = true + efa_indices = [0, 4, 8, 12] min_size = 2 max_size = 2 desired_size = 2 + + labels = { + "vpc.amazonaws.com/efa.present" = "true" + "nvidia.com/gpu.present" = "true" + } + + taints = { + # Ensure only GPU workloads are scheduled on this node group + gpu = { + key = "nvidia.com/gpu" + value = "true" + effect = "NO_SCHEDULE" + } + } } } @@ -354,7 +394,7 @@ data "aws_ami" "eks_default" { filter { name = "name" - values = ["amazon-eks-node-${local.cluster_version}-v*"] + values = ["amazon-eks-node-al2023-x86_64-standard-${local.cluster_version}-v*"] } } From d2e6262188aa706c8315ddcd1cc23ae2749edca9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 2 Nov 2024 13:33:40 +0000 Subject: [PATCH 838/962] chore(release): version 20.28.0 [skip ci] ## [20.28.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.27.0...v20.28.0) (2024-11-02) ### Features * Add support for creating `efa-only` network interfaces ([#3196](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3196)) ([c6da22c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c6da22c78f60a8643a6c76f97c93724f4e1f4e5a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6169e0c148..4c0c115d20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.28.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.27.0...v20.28.0) (2024-11-02) + + +### Features + +* Add support for creating `efa-only` network interfaces ([#3196](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3196)) ([c6da22c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c6da22c78f60a8643a6c76f97c93724f4e1f4e5a)) + ## [20.27.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.26.1...v20.27.0) (2024-11-01) From a224334fc8000dc8728971dff8adad46ceb7a8a1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 8 Nov 2024 21:56:56 +0000 Subject: [PATCH 839/962] feat: Add support for pod identity association on EKS addons (#3203) --- README.md | 5 ++-- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 6 ++--- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- main.tf | 28 +++++++++++++++++--- modules/eks-managed-node-group/README.md | 4 +-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 +-- modules/fargate-profile/versions.tf | 2 +- modules/karpenter/README.md | 4 +-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 +-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 5 ++-- tests/eks-managed-node-group/main.tf | 26 +++++++++++++++--- tests/eks-managed-node-group/versions.tf | 2 +- tests/fargate-profile/README.md | 4 +-- tests/fargate-profile/versions.tf | 2 +- tests/self-managed-node-group/README.md | 5 ++-- tests/self-managed-node-group/main.tf | 19 +++++++++++++ tests/self-managed-node-group/versions.tf | 2 +- versions.tf | 2 +- 23 files changed, 97 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index b272a6475d..5a074b966d 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,6 @@ module "eks" { access_entries = { # One access entry with a policy associated example = { - kubernetes_groups = [] principal_arn = "arn:aws:iam::123456789012:role/something" policy_associations = { @@ -175,7 +174,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.74 | +| [aws](#requirement\_aws) | >= 5.75 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -183,7 +182,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.74 | +| [aws](#provider\_aws) | >= 5.75 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index 3cc97fa038..0099e6baaf 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index b621a36591..15d51bcdb9 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -89,7 +89,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.74 | +| [aws](#requirement\_aws) | >= 5.75 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -97,8 +97,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.74 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.74 | +| [aws](#provider\_aws) | >= 5.75 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.75 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 0c0cc6c763..5caab8394a 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index 3cc97fa038..0099e6baaf 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } } } diff --git a/main.tf b/main.tf index 0b62869c24..037de7b5d8 100644 --- a/main.tf +++ b/main.tf @@ -496,8 +496,18 @@ resource "aws_eks_addon" "this" { cluster_name = aws_eks_cluster.this[0].name addon_name = try(each.value.name, each.key) - addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) - configuration_values = try(each.value.configuration_values, null) + addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) + configuration_values = try(each.value.configuration_values, null) + + dynamic "pod_identity_association" { + for_each = try(each.value.pod_identity_association, []) + + content { + role_arn = pod_identity_association.value.role_arn + service_account = pod_identity_association.value.service_account + } + } + preserve = try(each.value.preserve, true) resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, "OVERWRITE") resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE") @@ -525,8 +535,18 @@ resource "aws_eks_addon" "before_compute" { cluster_name = aws_eks_cluster.this[0].name addon_name = try(each.value.name, each.key) - addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) - configuration_values = try(each.value.configuration_values, null) + addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) + configuration_values = try(each.value.configuration_values, null) + + dynamic "pod_identity_association" { + for_each = try(each.value.pod_identity_association, []) + + content { + role_arn = pod_identity_association.value.role_arn + service_account = pod_identity_association.value.service_account + } + } + preserve = try(each.value.preserve, true) resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, "OVERWRITE") resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE") diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index ace9106c3d..23df973444 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.74 | +| [aws](#requirement\_aws) | >= 5.75 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.74 | +| [aws](#provider\_aws) | >= 5.75 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 3cc97fa038..0099e6baaf 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 732cd7b455..a7b12553ff 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.74 | +| [aws](#requirement\_aws) | >= 5.75 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.74 | +| [aws](#provider\_aws) | >= 5.75 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 3cc97fa038..0099e6baaf 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 2ca911d156..ef2be2099c 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.74 | +| [aws](#requirement\_aws) | >= 5.75 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.74 | +| [aws](#provider\_aws) | >= 5.75 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 3cc97fa038..0099e6baaf 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index d2c53be59a..7c76477049 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.74 | +| [aws](#requirement\_aws) | >= 5.75 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.74 | +| [aws](#provider\_aws) | >= 5.75 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 3cc97fa038..0099e6baaf 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } } } diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 39af8f08ed..b1f4ab1398 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,18 +18,19 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.74 | +| [aws](#requirement\_aws) | >= 5.75 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.74 | +| [aws](#provider\_aws) | >= 5.75 | ## Modules | Name | Source | Version | |------|--------|---------| +| [aws\_vpc\_cni\_ipv6\_pod\_identity](#module\_aws\_vpc\_cni\_ipv6\_pod\_identity) | terraform-aws-modules/eks-pod-identity/aws | ~> 1.6 | | [disabled\_eks](#module\_disabled\_eks) | ../.. | n/a | | [disabled\_eks\_managed\_node\_group](#module\_disabled\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | | [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 2.1 | diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index 7292e765dc..8a4d48c7f0 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -45,6 +45,10 @@ module "eks" { coredns = { most_recent = true } + eks-pod-identity-agent = { + before_compute = true + most_recent = true + } kube-proxy = { most_recent = true } @@ -58,6 +62,10 @@ module "eks" { WARM_PREFIX_TARGET = "1" } }) + pod_identity_association = [{ + role_arn = module.aws_vpc_cni_ipv6_pod_identity.iam_role_arn + service_account = "aws-node" + }] } } @@ -366,8 +374,7 @@ module "eks" { access_entries = { # One access entry with a policy associated ex-single = { - kubernetes_groups = [] - principal_arn = aws_iam_role.this["single"].arn + principal_arn = aws_iam_role.this["single"].arn policy_associations = { single = { @@ -382,8 +389,7 @@ module "eks" { # Example of adding multiple policies to a single access entry ex-multiple = { - kubernetes_groups = [] - principal_arn = aws_iam_role.this["multiple"].arn + principal_arn = aws_iam_role.this["multiple"].arn policy_associations = { ex-one = { @@ -489,6 +495,18 @@ module "vpc" { tags = local.tags } +module "aws_vpc_cni_ipv6_pod_identity" { + source = "terraform-aws-modules/eks-pod-identity/aws" + version = "~> 1.6" + + name = "aws-vpc-cni-ipv6" + + attach_aws_vpc_cni_policy = true + aws_vpc_cni_enable_ipv6 = true + + tags = local.tags +} + module "ebs_kms_key" { source = "terraform-aws-modules/kms/aws" version = "~> 2.1" diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index 3cc97fa038..0099e6baaf 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } } } diff --git a/tests/fargate-profile/README.md b/tests/fargate-profile/README.md index a7af7cd080..a50029c722 100644 --- a/tests/fargate-profile/README.md +++ b/tests/fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.74 | +| [aws](#requirement\_aws) | >= 5.75 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.74 | +| [aws](#provider\_aws) | >= 5.75 | ## Modules diff --git a/tests/fargate-profile/versions.tf b/tests/fargate-profile/versions.tf index 3cc97fa038..0099e6baaf 100644 --- a/tests/fargate-profile/versions.tf +++ b/tests/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 0691b7aa0b..1587f7c177 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,18 +18,19 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.74 | +| [aws](#requirement\_aws) | >= 5.75 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.74 | +| [aws](#provider\_aws) | >= 5.75 | ## Modules | Name | Source | Version | |------|--------|---------| +| [aws\_vpc\_cni\_ipv4\_pod\_identity](#module\_aws\_vpc\_cni\_ipv4\_pod\_identity) | terraform-aws-modules/eks-pod-identity/aws | ~> 1.6 | | [disabled\_self\_managed\_node\_group](#module\_disabled\_self\_managed\_node\_group) | ../../modules/self-managed-node-group | n/a | | [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 2.0 | | [eks](#module\_eks) | ../.. | n/a | diff --git a/tests/self-managed-node-group/main.tf b/tests/self-managed-node-group/main.tf index dee3274dc4..afe7aac9a1 100644 --- a/tests/self-managed-node-group/main.tf +++ b/tests/self-managed-node-group/main.tf @@ -41,11 +41,18 @@ module "eks" { coredns = { most_recent = true } + eks-pod-identity-agent = { + most_recent = true + } kube-proxy = { most_recent = true } vpc-cni = { most_recent = true + pod_identity_association = [{ + role_arn = module.aws_vpc_cni_ipv4_pod_identity.iam_role_arn + service_account = "aws-node" + }] } } @@ -388,6 +395,18 @@ module "vpc" { tags = local.tags } +module "aws_vpc_cni_ipv4_pod_identity" { + source = "terraform-aws-modules/eks-pod-identity/aws" + version = "~> 1.6" + + name = "aws-vpc-cni-ipv4" + + attach_aws_vpc_cni_policy = true + aws_vpc_cni_enable_ipv4 = true + + tags = local.tags +} + data "aws_ami" "eks_default" { most_recent = true owners = ["amazon"] diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index 3cc97fa038..0099e6baaf 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } } } diff --git a/versions.tf b/versions.tf index 090ca7b00b..fc9dadd253 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.74" + version = ">= 5.75" } tls = { source = "hashicorp/tls" From 97a08c8aff5dbf51a86b4c8cd88a858336cd0208 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 8 Nov 2024 21:57:23 +0000 Subject: [PATCH 840/962] chore(release): version 20.29.0 [skip ci] ## [20.29.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.28.0...v20.29.0) (2024-11-08) ### Features * Add support for pod identity association on EKS addons ([#3203](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3203)) ([a224334](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a224334fc8000dc8728971dff8adad46ceb7a8a1)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c0c115d20..d8aee823f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.29.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.28.0...v20.29.0) (2024-11-08) + + +### Features + +* Add support for pod identity association on EKS addons ([#3203](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3203)) ([a224334](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a224334fc8000dc8728971dff8adad46ceb7a8a1)) + ## [20.28.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.27.0...v20.28.0) (2024-11-02) From ab2207d50949079d5dd97c976c6f7a8f5b668f0c Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 26 Nov 2024 13:31:42 -0600 Subject: [PATCH 841/962] feat: Improve addon dependency chain and decrease time to provision addons (due to retries) (#3218) * feat: Improve addon dependency chain and decrease time to provision addons (due to retries) * fix: Run pre-commit to clean up docs --- main.tf | 29 +++-- node_groups.tf | 2 +- tests/fast-addons/README.md | 92 ++++++++++++++ tests/fast-addons/main.tf | 159 +++++++++++++++++++++++ tests/fast-addons/outputs.tf | 226 +++++++++++++++++++++++++++++++++ tests/fast-addons/variables.tf | 0 tests/fast-addons/versions.tf | 10 ++ variables.tf | 1 + 8 files changed, 508 insertions(+), 11 deletions(-) create mode 100644 tests/fast-addons/README.md create mode 100644 tests/fast-addons/main.tf create mode 100644 tests/fast-addons/outputs.tf create mode 100644 tests/fast-addons/variables.tf create mode 100644 tests/fast-addons/versions.tf diff --git a/main.tf b/main.tf index 037de7b5d8..0046aa0c6b 100644 --- a/main.tf +++ b/main.tf @@ -208,7 +208,7 @@ locals { resource "aws_eks_access_entry" "this" { for_each = { for k, v in local.merged_access_entries : k => v if local.create } - cluster_name = aws_eks_cluster.this[0].name + cluster_name = aws_eks_cluster.this[0].id kubernetes_groups = try(each.value.kubernetes_groups, null) principal_arn = each.value.principal_arn type = try(each.value.type, "STANDARD") @@ -225,7 +225,7 @@ resource "aws_eks_access_policy_association" "this" { type = each.value.association_access_scope_type } - cluster_name = aws_eks_cluster.this[0].name + cluster_name = aws_eks_cluster.this[0].id policy_arn = each.value.association_policy_arn principal_arn = each.value.principal_arn @@ -481,19 +481,25 @@ resource "aws_iam_policy" "cluster_encryption" { # EKS Addons ################################################################################ +locals { + # TODO - Set to `NONE` on next breaking change when default addons are disabled + resolve_conflicts_on_create_default = var.bootstrap_self_managed_addons ? "OVERWRITE" : "NONE" +} + data "aws_eks_addon_version" "this" { for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } addon_name = try(each.value.name, each.key) kubernetes_version = coalesce(var.cluster_version, aws_eks_cluster.this[0].version) - most_recent = try(each.value.most_recent, null) + # TODO - Set default fallback to `true` on next breaking change + most_recent = try(each.value.most_recent, null) } resource "aws_eks_addon" "this" { # Not supported on outposts for_each = { for k, v in var.cluster_addons : k => v if !try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } - cluster_name = aws_eks_cluster.this[0].name + cluster_name = aws_eks_cluster.this[0].id addon_name = try(each.value.name, each.key) addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) @@ -508,8 +514,9 @@ resource "aws_eks_addon" "this" { } } - preserve = try(each.value.preserve, true) - resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, "OVERWRITE") + preserve = try(each.value.preserve, true) + # TODO - Set to `NONE` on next breaking change when default addons are disabled + resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, local.resolve_conflicts_on_create_default) resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE") service_account_role_arn = try(each.value.service_account_role_arn, null) @@ -532,7 +539,7 @@ resource "aws_eks_addon" "before_compute" { # Not supported on outposts for_each = { for k, v in var.cluster_addons : k => v if try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } - cluster_name = aws_eks_cluster.this[0].name + cluster_name = aws_eks_cluster.this[0].id addon_name = try(each.value.name, each.key) addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) @@ -547,8 +554,9 @@ resource "aws_eks_addon" "before_compute" { } } - preserve = try(each.value.preserve, true) - resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, "OVERWRITE") + preserve = try(each.value.preserve, true) + # TODO - Set to `NONE` on next breaking change when default addons are disabled + resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, local.resolve_conflicts_on_create_default) resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE") service_account_role_arn = try(each.value.service_account_role_arn, null) @@ -570,6 +578,7 @@ locals { # Maintain current behavior for <= 1.29, remove default for >= 1.30 # `null` will return the latest Kubernetes version from the EKS API, which at time of writing is 1.30 # https://github.com/kubernetes/kubernetes/pull/123561 + # TODO - remove on next breaking change in conjunction with issuer URL change below idpc_backwards_compat_version = contains(["1.21", "1.22", "1.23", "1.24", "1.25", "1.26", "1.27", "1.28", "1.29"], coalesce(var.cluster_version, "1.30")) idpc_issuer_url = local.idpc_backwards_compat_version ? try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, null) : null } @@ -577,7 +586,7 @@ locals { resource "aws_eks_identity_provider_config" "this" { for_each = { for k, v in var.cluster_identity_providers : k => v if local.create && !local.create_outposts_local_cluster } - cluster_name = aws_eks_cluster.this[0].name + cluster_name = aws_eks_cluster.this[0].id oidc { client_id = each.value.client_id diff --git a/node_groups.tf b/node_groups.tf index a8e499abfa..d18101253b 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -32,7 +32,7 @@ resource "time_sleep" "this" { create_duration = var.dataplane_wait_duration triggers = { - cluster_name = aws_eks_cluster.this[0].name + cluster_name = aws_eks_cluster.this[0].id cluster_endpoint = aws_eks_cluster.this[0].endpoint cluster_version = aws_eks_cluster.this[0].version cluster_service_cidr = var.cluster_ip_family == "ipv6" ? try(local.kubernetes_network_config.service_ipv6_cidr, "") : try(local.kubernetes_network_config.service_ipv4_cidr, "") diff --git a/tests/fast-addons/README.md b/tests/fast-addons/README.md new file mode 100644 index 0000000000..693784e79d --- /dev/null +++ b/tests/fast-addons/README.md @@ -0,0 +1,92 @@ +# Fast Addons + +Refer to https://github.com/terraform-aws-modules/terraform-aws-eks/pull/3214 for additional information. + + + +## Usage + +To provision the provided configurations you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply --auto-approve +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [aws](#requirement\_aws) | >= 5.75 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.75 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | n/a | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_route_table_association.custom_network](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association) | resource | +| [aws_subnet.custom_network](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource | +| [aws_vpc_ipv4_cidr_block_association.custom_network](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_ipv4_cidr_block_association) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | +| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | +| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | +| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | +| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | +| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | +| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | +| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | +| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | +| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | +| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | +| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | +| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | +| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | +| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | +| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | +| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | +| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | +| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | +| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | + diff --git a/tests/fast-addons/main.tf b/tests/fast-addons/main.tf new file mode 100644 index 0000000000..12d2a41f8c --- /dev/null +++ b/tests/fast-addons/main.tf @@ -0,0 +1,159 @@ +provider "aws" { + region = local.region +} + +locals { + name = "ex-${basename(path.cwd)}" + cluster_version = "1.31" + region = "eu-west-1" + + tags = { + Test = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true + + enable_cluster_creator_admin_permissions = true + + # Disable the default self-managed addons to avoid the penalty of adopting them later + bootstrap_self_managed_addons = false + + # Addons will be provisioned net new via the EKS addon API + cluster_addons = { + coredns = { + most_recent = true + } + eks-pod-identity-agent = { + before_compute = true + most_recent = true + } + kube-proxy = { + most_recent = true + } + vpc-cni = { + most_recent = true + before_compute = true + configuration_values = jsonencode({ + env = { + # Use subnet tags to avoid the need to inject the ENIConfig + # which requires a live API server endpoint which leads to a dependency of: + # Control plane -> API request to create ENIConfig -> VPC CNI addon -> nodes/compute + # With the subnet discovery feature, we can avoid this dependency: + # Control plane -> VPC CNI addon -> nodes/compute + ENABLE_SUBNET_DISCOVERY = "true" + } + }) + } + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + example = { + instance_types = ["m6i.large"] + + min_size = 2 + max_size = 5 + desired_size = 2 + } + } + + tags = local.tags +} + +################################################################################ +# VPC +################################################################################ + +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + +locals { + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + tags = local.tags +} + +################################################################################ +# Custom Networking +################################################################################ + +locals { + custom_network_vpc_cidr = "10.99.0.0/16" + + custom_network_subnets = [for k, v in local.azs : cidrsubnet(local.custom_network_vpc_cidr, 4, k)] +} + +resource "aws_vpc_ipv4_cidr_block_association" "custom_network" { + vpc_id = module.vpc.vpc_id + cidr_block = local.custom_network_vpc_cidr +} + +resource "aws_subnet" "custom_network" { + count = length(local.custom_network_subnets) + + vpc_id = module.vpc.vpc_id + cidr_block = element(local.custom_network_subnets, count.index) + + tags = merge( + local.tags, + { + # Tag for subnet discovery + "kubernetes.io/role/cni" = 1 + "kubernetes.io/role/internal-elb" = 1 + } + ) + + depends_on = [ + aws_vpc_ipv4_cidr_block_association.custom_network + ] +} + +resource "aws_route_table_association" "custom_network" { + count = length(local.custom_network_subnets) + + subnet_id = element(aws_subnet.custom_network[*].id, count.index) + route_table_id = element(module.vpc.private_route_table_ids, 0) + + depends_on = [ + aws_vpc_ipv4_cidr_block_association.custom_network + ] +} diff --git a/tests/fast-addons/outputs.tf b/tests/fast-addons/outputs.tf new file mode 100644 index 0000000000..9357464c29 --- /dev/null +++ b/tests/fast-addons/outputs.tf @@ -0,0 +1,226 @@ +################################################################################ +# Cluster +################################################################################ + +output "cluster_arn" { + description = "The Amazon Resource Name (ARN) of the cluster" + value = module.eks.cluster_arn +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded certificate data required to communicate with the cluster" + value = module.eks.cluster_certificate_authority_data +} + +output "cluster_endpoint" { + description = "Endpoint for your Kubernetes API server" + value = module.eks.cluster_endpoint +} + +output "cluster_id" { + description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" + value = module.eks.cluster_id +} + +output "cluster_name" { + description = "The name of the EKS cluster" + value = module.eks.cluster_name +} + +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_oidc_issuer_url +} + +output "cluster_dualstack_oidc_issuer_url" { + description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_dualstack_oidc_issuer_url +} + +output "cluster_platform_version" { + description = "Platform version for the cluster" + value = module.eks.cluster_platform_version +} + +output "cluster_status" { + description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" + value = module.eks.cluster_status +} + +output "cluster_primary_security_group_id" { + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = module.eks.cluster_primary_security_group_id +} + +output "cluster_service_cidr" { + description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" + value = module.eks.cluster_service_cidr +} + +output "cluster_ip_family" { + description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" + value = module.eks.cluster_ip_family +} + +################################################################################ +# Access Entry +################################################################################ + +output "access_entries" { + description = "Map of access entries created and their attributes" + value = module.eks.access_entries +} + +################################################################################ +# KMS Key +################################################################################ + +output "kms_key_arn" { + description = "The Amazon Resource Name (ARN) of the key" + value = module.eks.kms_key_arn +} + +output "kms_key_id" { + description = "The globally unique identifier for the key" + value = module.eks.kms_key_id +} + +output "kms_key_policy" { + description = "The IAM resource policy set on the key" + value = module.eks.kms_key_policy +} + +################################################################################ +# Security Group +################################################################################ + +output "cluster_security_group_arn" { + description = "Amazon Resource Name (ARN) of the cluster security group" + value = module.eks.cluster_security_group_arn +} + +output "cluster_security_group_id" { + description = "ID of the cluster security group" + value = module.eks.cluster_security_group_id +} + +################################################################################ +# Node Security Group +################################################################################ + +output "node_security_group_arn" { + description = "Amazon Resource Name (ARN) of the node shared security group" + value = module.eks.node_security_group_arn +} + +output "node_security_group_id" { + description = "ID of the node shared security group" + value = module.eks.node_security_group_id +} + +################################################################################ +# IRSA +################################################################################ + +output "oidc_provider" { + description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" + value = module.eks.oidc_provider +} + +output "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + value = module.eks.oidc_provider_arn +} + +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = module.eks.cluster_tls_certificate_sha1_fingerprint +} + +################################################################################ +# IAM Role +################################################################################ + +output "cluster_iam_role_name" { + description = "IAM role name of the EKS cluster" + value = module.eks.cluster_iam_role_name +} + +output "cluster_iam_role_arn" { + description = "IAM role ARN of the EKS cluster" + value = module.eks.cluster_iam_role_arn +} + +output "cluster_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.cluster_iam_role_unique_id +} + +################################################################################ +# EKS Addons +################################################################################ + +output "cluster_addons" { + description = "Map of attribute maps for all EKS cluster addons enabled" + value = module.eks.cluster_addons +} + +################################################################################ +# EKS Identity Provider +################################################################################ + +output "cluster_identity_providers" { + description = "Map of attribute maps for all EKS identity providers enabled" + value = module.eks.cluster_identity_providers +} + +################################################################################ +# CloudWatch Log Group +################################################################################ + +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_name +} + +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_arn +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.eks.fargate_profiles +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +output "eks_managed_node_groups" { + description = "Map of attribute maps for all EKS managed node groups created" + value = module.eks.eks_managed_node_groups +} + +output "eks_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by EKS managed node groups" + value = module.eks.eks_managed_node_groups_autoscaling_group_names +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +output "self_managed_node_groups" { + description = "Map of attribute maps for all self managed node groups created" + value = module.eks.self_managed_node_groups +} + +output "self_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by self-managed node groups" + value = module.eks.self_managed_node_groups_autoscaling_group_names +} diff --git a/tests/fast-addons/variables.tf b/tests/fast-addons/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/fast-addons/versions.tf b/tests/fast-addons/versions.tf new file mode 100644 index 0000000000..0099e6baaf --- /dev/null +++ b/tests/fast-addons/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.2" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.75" + } + } +} diff --git a/variables.tf b/variables.tf index 7a7226b96a..edd0d938a6 100644 --- a/variables.tf +++ b/variables.tf @@ -148,6 +148,7 @@ variable "cluster_timeouts" { default = {} } +# TODO - hard code to false on next breaking change variable "bootstrap_self_managed_addons" { description = "Indicates whether or not to bootstrap self-managed addons after the cluster has been created" type = bool From b468ead1d97d3c9a83fad227884b3cc689f2b200 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 26 Nov 2024 19:32:13 +0000 Subject: [PATCH 842/962] chore(release): version 20.30.0 [skip ci] ## [20.30.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.29.0...v20.30.0) (2024-11-26) ### Features * Improve addon dependency chain and decrease time to provision addons (due to retries) ([#3218](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3218)) ([ab2207d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ab2207d50949079d5dd97c976c6f7a8f5b668f0c)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8aee823f6..cc895c9dd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.30.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.29.0...v20.30.0) (2024-11-26) + + +### Features + +* Improve addon dependency chain and decrease time to provision addons (due to retries) ([#3218](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3218)) ([ab2207d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ab2207d50949079d5dd97c976c6f7a8f5b668f0c)) + ## [20.29.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.28.0...v20.29.0) (2024-11-08) From 35388bb8c4cfa0c351427c133490b914b9944b07 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 26 Nov 2024 14:35:31 -0600 Subject: [PATCH 843/962] fix: Coalesce local `resolve_conflicts_on_create_default` value to a boolean since default is `null` (#3221) --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 0046aa0c6b..50f4323c24 100644 --- a/main.tf +++ b/main.tf @@ -483,7 +483,7 @@ resource "aws_iam_policy" "cluster_encryption" { locals { # TODO - Set to `NONE` on next breaking change when default addons are disabled - resolve_conflicts_on_create_default = var.bootstrap_self_managed_addons ? "OVERWRITE" : "NONE" + resolve_conflicts_on_create_default = coalesce(var.bootstrap_self_managed_addons, true) ? "OVERWRITE" : "NONE" } data "aws_eks_addon_version" "this" { From 6866b40becaddeca60638096da919732b8b52902 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 26 Nov 2024 20:36:02 +0000 Subject: [PATCH 844/962] chore(release): version 20.30.1 [skip ci] ## [20.30.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.30.0...v20.30.1) (2024-11-26) ### Bug Fixes * Coalesce local `resolve_conflicts_on_create_default` value to a boolean since default is `null` ([#3221](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3221)) ([35388bb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/35388bb8c4cfa0c351427c133490b914b9944b07)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc895c9dd4..2c7b9ec0ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.30.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.30.0...v20.30.1) (2024-11-26) + + +### Bug Fixes + +* Coalesce local `resolve_conflicts_on_create_default` value to a boolean since default is `null` ([#3221](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3221)) ([35388bb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/35388bb8c4cfa0c351427c133490b914b9944b07)) + ## [20.30.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.29.0...v20.30.0) (2024-11-26) From 3b974d33ad79e142566dd7bcb4bf10472cc91899 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 4 Dec 2024 09:24:21 -0600 Subject: [PATCH 845/962] feat: Add support for EKS Auto Mode and EKS Hybrid nodes (#3225) * feat: Add support for EKS hybrid nodes * feat: Add support for EKS Auto Mode * chore: Update test directory names * chore: Clean up examples and tests * fix: Clean up and last minute changes for GA * chore: Formatting * chore: Bump min required version for new features * fix: Corrects from test/validation on existing clusters * feat: Add policy for custom tags on EKS Auto Mode, validate examples * chore: Expand on `CAM` acronym * chore: Update README to match examples --- README.md | 227 +++++- examples/eks-auto-mode/README.md | 96 +++ examples/eks-auto-mode/deployment.yaml | 21 + examples/eks-auto-mode/main.tf | 86 +++ .../eks-auto-mode}/outputs.tf | 23 +- .../eks-auto-mode}/variables.tf | 0 .../eks-auto-mode}/versions.tf | 2 +- examples/eks-hybrid-nodes/.gitignore | 2 + examples/eks-hybrid-nodes/README.md | 82 ++ .../ami/amazon-eks-ubuntu.pkr.hcl | 320 ++++++++ examples/eks-hybrid-nodes/ami/plugins.pkr.hcl | 8 + .../eks-hybrid-nodes/ami/variables.pkr.hcl | 723 ++++++++++++++++++ examples/eks-hybrid-nodes/main.tf | 148 ++++ examples/eks-hybrid-nodes/outputs.tf | 0 examples/eks-hybrid-nodes/remote.tf | 314 ++++++++ examples/eks-hybrid-nodes/variables.tf | 0 examples/eks-hybrid-nodes/versions.tf | 26 + examples/eks-managed-node-group/main.tf | 8 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 55 +- examples/karpenter/main.tf | 9 +- examples/karpenter/outputs.tf | 297 ------- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/main.tf | 8 +- examples/self-managed-node-group/versions.tf | 2 +- main.tf | 320 +++++++- modules/eks-managed-node-group/README.md | 4 +- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 +- modules/fargate-profile/versions.tf | 2 +- modules/hybrid-node-role/README.md | 159 ++++ modules/hybrid-node-role/main.tf | 369 +++++++++ modules/hybrid-node-role/outputs.tf | 37 + modules/hybrid-node-role/variables.tf | 239 ++++++ modules/hybrid-node-role/versions.tf | 10 + modules/karpenter/README.md | 4 +- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 +- modules/self-managed-node-group/versions.tf | 2 +- outputs.tf | 23 +- .../README.md | 13 +- .../main.tf | 8 +- tests/eks-fargate-profile/outputs.tf | 245 ++++++ tests/eks-fargate-profile/variables.tf | 0 tests/eks-fargate-profile/versions.tf | 10 + tests/eks-hybrid-nodes/README.md | 65 ++ tests/eks-hybrid-nodes/main.tf | 84 ++ tests/eks-hybrid-nodes/outputs.tf | 71 ++ tests/eks-hybrid-nodes/variables.tf | 0 tests/eks-hybrid-nodes/versions.tf | 14 + tests/eks-managed-node-group/README.md | 11 +- tests/eks-managed-node-group/main.tf | 9 +- tests/eks-managed-node-group/outputs.tf | 23 +- tests/eks-managed-node-group/versions.tf | 2 +- tests/fast-addons/README.md | 4 +- tests/fast-addons/versions.tf | 2 +- tests/self-managed-node-group/README.md | 11 +- tests/self-managed-node-group/main.tf | 9 +- tests/self-managed-node-group/outputs.tf | 23 +- tests/self-managed-node-group/versions.tf | 2 +- variables.tf | 87 ++- versions.tf | 2 +- 62 files changed, 3896 insertions(+), 441 deletions(-) create mode 100644 examples/eks-auto-mode/README.md create mode 100644 examples/eks-auto-mode/deployment.yaml create mode 100644 examples/eks-auto-mode/main.tf rename {tests/fargate-profile => examples/eks-auto-mode}/outputs.tf (92%) rename {tests/fargate-profile => examples/eks-auto-mode}/variables.tf (100%) rename {tests/fargate-profile => examples/eks-auto-mode}/versions.tf (82%) create mode 100644 examples/eks-hybrid-nodes/.gitignore create mode 100644 examples/eks-hybrid-nodes/README.md create mode 100644 examples/eks-hybrid-nodes/ami/amazon-eks-ubuntu.pkr.hcl create mode 100644 examples/eks-hybrid-nodes/ami/plugins.pkr.hcl create mode 100644 examples/eks-hybrid-nodes/ami/variables.pkr.hcl create mode 100644 examples/eks-hybrid-nodes/main.tf create mode 100644 examples/eks-hybrid-nodes/outputs.tf create mode 100644 examples/eks-hybrid-nodes/remote.tf create mode 100644 examples/eks-hybrid-nodes/variables.tf create mode 100644 examples/eks-hybrid-nodes/versions.tf create mode 100644 modules/hybrid-node-role/README.md create mode 100644 modules/hybrid-node-role/main.tf create mode 100644 modules/hybrid-node-role/outputs.tf create mode 100644 modules/hybrid-node-role/variables.tf create mode 100644 modules/hybrid-node-role/versions.tf rename tests/{fargate-profile => eks-fargate-profile}/README.md (91%) rename tests/{fargate-profile => eks-fargate-profile}/main.tf (96%) create mode 100644 tests/eks-fargate-profile/outputs.tf create mode 100644 tests/eks-fargate-profile/variables.tf create mode 100644 tests/eks-fargate-profile/versions.tf create mode 100644 tests/eks-hybrid-nodes/README.md create mode 100644 tests/eks-hybrid-nodes/main.tf create mode 100644 tests/eks-hybrid-nodes/outputs.tf create mode 100644 tests/eks-hybrid-nodes/variables.tf create mode 100644 tests/eks-hybrid-nodes/versions.tf diff --git a/README.md b/README.md index 5a074b966d..8f7cc2727b 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,124 @@ Please note that we strive to provide a comprehensive suite of documentation for ## Usage +### EKS Auto Mode + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.31" + + cluster_name = "example" + cluster_version = "1.31" + + # Optional + cluster_endpoint_public_access = true + + # Optional: Adds the current caller identity as an administrator via cluster access entry + enable_cluster_creator_admin_permissions = true + + cluster_compute_config = { + enabled = true + node_pools = ["general-purpose"] + } + + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### EKS Hybrid Nodes + +```hcl +locals { + # RFC 1918 IP ranges supported + remote_network_cidr = "172.16.0.0/16" + remote_node_cidr = cidrsubnet(local.remote_network_cidr, 2, 0) + remote_pod_cidr = cidrsubnet(local.remote_network_cidr, 2, 1) +} + +# SSM and IAM Roles Anywhere supported - SSM is default +module "eks_hybrid_node_role" { + source = "terraform-aws-modules/eks/aws//modules/hybrid-node-role" + version = "~> 20.31" + + tags = { + Environment = "dev" + Terraform = "true" + } +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.31" + + cluster_name = "example" + cluster_version = "1.31" + + cluster_addons = { + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + } + + # Optional + cluster_endpoint_public_access = true + + # Optional: Adds the current caller identity as an administrator via cluster access entry + enable_cluster_creator_admin_permissions = true + + create_node_security_group = false + cluster_security_group_additional_rules = { + hybrid-all = { + cidr_blocks = [local.remote_network_cidr] + description = "Allow all traffic from remote node/pod network" + from_port = 0 + to_port = 0 + protocol = "all" + type = "ingress" + } + } + + # Optional + cluster_compute_config = { + enabled = true + node_pools = ["system"] + } + + access_entries = { + hybrid-node-role = { + principal_arn = module.eks_hybrid_node_role.arn + type = "HYBRID_LINUX" + } + } + + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + + cluster_remote_network_config = { + remote_node_networks = { + cidrs = [local.remote_node_cidr] + } + # Required if running webhooks on Hybrid nodes + remote_pod_networks = { + cidrs = [local.remote_pod_cidr] + } + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### EKS Managed Node Group + ```hcl module "eks" { source = "terraform-aws-modules/eks/aws" @@ -33,8 +151,7 @@ module "eks" { cluster_name = "my-cluster" cluster_version = "1.31" - cluster_endpoint_public_access = true - + bootstrap_self_managed_addons = false cluster_addons = { coredns = {} eks-pod-identity-agent = {} @@ -42,6 +159,12 @@ module "eks" { vpc-cni = {} } + # Optional + cluster_endpoint_public_access = true + + # Optional: Adds the current caller identity as an administrator via cluster access entry + enable_cluster_creator_admin_permissions = true + vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] control_plane_subnet_ids = ["subnet-xyzde987", "subnet-slkjf456", "subnet-qeiru789"] @@ -63,14 +186,30 @@ module "eks" { } } - # Cluster access entry - # To add the current caller identity as an administrator - enable_cluster_creator_admin_permissions = true + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### Cluster Access Entry + +When enabling `authentication_mode = "API_AND_CONFIG_MAP"`, EKS will automatically create an access entry for the IAM role(s) used by managed node group(s) and Fargate profile(s). There are no additional actions required by users. For self-managed node groups and the Karpenter sub-module, this project automatically adds the access entry on behalf of users so there are no additional actions required by users. + +On clusters that were created prior to cluster access management (CAM) support, there will be an existing access entry for the cluster creator. This was previously not visible when using `aws-auth` ConfigMap, but will become visible when access entry is enabled. + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + # Truncated for brevity ... access_entries = { # One access entry with a policy associated example = { - principal_arn = "arn:aws:iam::123456789012:role/something" + principal_arn = "arn:aws:iam::123456789012:role/something" policy_associations = { example = { @@ -83,20 +222,9 @@ module "eks" { } } } - - tags = { - Environment = "dev" - Terraform = "true" - } } ``` -### Cluster Access Entry - -When enabling `authentication_mode = "API_AND_CONFIG_MAP"`, EKS will automatically create an access entry for the IAM role(s) used by managed node group(s) and Fargate profile(s). There are no additional actions required by users. For self-managed node groups and the Karpenter sub-module, this project automatically adds the access entry on behalf of users so there are no additional actions required by users. - -On clusters that were created prior to CAM support, there will be an existing access entry for the cluster creator. This was previously not visible when using `aws-auth` ConfigMap, but will become visible when access entry is enabled. - ### Bootstrap Cluster Creator Admin Permissions Setting the `bootstrap_cluster_creator_admin_permissions` is a one time operation when the cluster is created; it cannot be modified later through the EKS API. In this project we are hardcoding this to `false`. If users wish to achieve the same functionality, we will do that through an access entry which can be enabled or disabled at any time of their choosing using the variable `enable_cluster_creator_admin_permissions` @@ -133,17 +261,31 @@ module "eks" { eks_managed_node_groups = { example = { + # The EKS AL2023 NVIDIA AMI provides all of the necessary components + # for accelerated workloads w/ EFA + ami_type = "AL2023_x86_64_NVIDIA" instance_types = ["p5.48xlarge"] # Exposes all EFA interfaces on the launch template created by the node group(s) # This would expose all 32 EFA interfaces for the p5.48xlarge instance type enable_efa_support = true - pre_bootstrap_user_data = <<-EOT - # Mount NVME instance store volumes since they are typically - # available on instance types that support EFA - setup-local-disks raid0 - EOT + # Mount instance store volumes in RAID-0 for kubelet and containerd + # https://github.com/awslabs/amazon-eks-ami/blob/master/doc/USER_GUIDE.md#raid-0-for-kubelet-and-containerd-raid0 + cloudinit_pre_nodeadm = [ + { + content_type = "application/node.eks.aws" + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + instance: + localStorage: + strategy: RAID0 + EOT + } + ] # EFA should only be enabled when connecting 2 or more nodes # Do not use EFA on a single node workload @@ -157,9 +299,11 @@ module "eks" { ## Examples -- [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-managed-node-group): EKS Cluster using EKS managed node groups +- [EKS Auto Mode](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-auto-mode): EKS Cluster with EKS Auto Mode +- [EKS Hybrid Nodes](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-hybrid-nodes): EKS Cluster with EKS Hybrid nodes +- [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-managed-node-group): EKS Cluster with EKS managed node group(s) - [Karpenter](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/karpenter): EKS Cluster with [Karpenter](https://karpenter.sh/) provisioned for intelligent data plane management -- [Self Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self-managed-node-group): EKS Cluster using self-managed node groups +- [Self Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self-managed-node-group): EKS Cluster with self-managed node group(s) ## Contributing @@ -174,7 +318,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.75 | +| [aws](#requirement\_aws) | >= 5.79 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -182,7 +326,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.75 | +| [aws](#provider\_aws) | >= 5.79 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | @@ -210,9 +354,14 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_iam_openid_connect_provider.oidc_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource | | [aws_iam_policy.cluster_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.eks_auto_custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.eks_auto](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cluster_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.eks_auto](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.eks_auto_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.eks_auto_custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_security_group.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | @@ -223,6 +372,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.eks_auto_custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.node_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_session_context.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_session_context) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [tls_certificate.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/data-sources/certificate) | data source | @@ -242,6 +393,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | | [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no | | [cluster\_addons\_timeouts](#input\_cluster\_addons\_timeouts) | Create, update, and delete timeout configurations for the cluster addons | `map(string)` | `{}` | no | +| [cluster\_compute\_config](#input\_cluster\_compute\_config) | Configuration block for the cluster compute configuration | `any` | `{}` | no | | [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | | [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster. To disable secret encryption, set this value to `{}` | `any` |
{
"resources": [
"secrets"
]
}
| no | | [cluster\_encryption\_policy\_description](#input\_cluster\_encryption\_policy\_description) | Description of the cluster encryption policy created | `string` | `"Cluster encryption policy to allow cluster role to utilize CMK provided"` | no | @@ -255,6 +407,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `"ipv4"` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | +| [cluster\_remote\_network\_config](#input\_cluster\_remote\_network\_config) | Configuration block for the cluster remote network configuration | `any` | `{}` | no | | [cluster\_security\_group\_additional\_rules](#input\_cluster\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created. Set `source_node_security_group = true` inside rules to set the `node_security_group` as source | `any` | `{}` | no | | [cluster\_security\_group\_description](#input\_cluster\_security\_group\_description) | Description of the cluster security group created | `string` | `"EKS cluster security group"` | no | | [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Existing security group ID to be attached to the cluster | `string` | `""` | no | @@ -274,8 +427,9 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [create\_cluster\_primary\_security\_group\_tags](#input\_create\_cluster\_primary\_security\_group\_tags) | Indicates whether or not to tag the cluster's primary security group. This security group is created by the EKS service, not the module, and therefore tagging is handled after cluster creation | `bool` | `true` | no | | [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default | `bool` | `true` | no | | [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy) | `bool` | `false` | no | -| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether a an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created for the cluster | `bool` | `true` | no | | [create\_kms\_key](#input\_create\_kms\_key) | Controls if a KMS key for cluster encryption should be created | `bool` | `true` | no | +| [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an EKS Auto node IAM role is created | `bool` | `true` | no | | [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | | [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | | [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed node group(s), self-managed node group(s), Fargate profile(s)) | `string` | `"30s"` | no | @@ -285,13 +439,15 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | | [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled | `bool` | `true` | no | +| [enable\_node\_custom\_tags\_permissions](#input\_enable\_node\_custom\_tags\_permissions) | Determines whether to enable permissions for custom tags for the EKS Auto node IAM role | `bool` | `true` | no | +| [enable\_security\_groups\_for\_pods](#input\_enable\_security\_groups\_for\_pods) | Determines whether to add the necessary IAM permission policy for security groups for pods | `bool` | `true` | no | | [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no | | [fargate\_profiles](#input\_fargate\_profiles) | Map of Fargate Profile definitions to create | `any` | `{}` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the cluster. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | -| [iam\_role\_path](#input\_iam\_role\_path) | Cluster IAM role path | `string` | `null` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | The IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | @@ -306,6 +462,14 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [kms\_key\_service\_users](#input\_kms\_key\_service\_users) | A list of IAM ARNs for [key service users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-service-integration) | `list(string)` | `[]` | no | | [kms\_key\_source\_policy\_documents](#input\_kms\_key\_source\_policy\_documents) | List of IAM policy documents that are merged together into the exported document. Statements must have unique `sid`s | `list(string)` | `[]` | no | | [kms\_key\_users](#input\_kms\_key\_users) | A list of IAM ARNs for [key users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-users) | `list(string)` | `[]` | no | +| [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the EKS Auto node IAM role | `map(string)` | `{}` | no | +| [node\_iam\_role\_description](#input\_node\_iam\_role\_description) | Description of the EKS Auto node IAM role | `string` | `null` | no | +| [node\_iam\_role\_name](#input\_node\_iam\_role\_name) | Name to use on the EKS Auto node IAM role created | `string` | `null` | no | +| [node\_iam\_role\_path](#input\_node\_iam\_role\_path) | The EKS Auto node IAM role path | `string` | `null` | no | +| [node\_iam\_role\_permissions\_boundary](#input\_node\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the EKS Auto node IAM role | `string` | `null` | no | +| [node\_iam\_role\_policy\_statements](#input\_node\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [node\_iam\_role\_tags](#input\_node\_iam\_role\_tags) | A map of additional tags to add to the EKS Auto node IAM role created | `map(string)` | `{}` | no | +| [node\_iam\_role\_use\_name\_prefix](#input\_node\_iam\_role\_use\_name\_prefix) | Determines whether the EKS Auto node IAM role name (`node_iam_role_name`) is used as a prefix | `bool` | `true` | no | | [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no | | [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | | [node\_security\_group\_enable\_recommended\_rules](#input\_node\_security\_group\_enable\_recommended\_rules) | Determines whether to enable recommended security group rules for the node security group created. This includes node-to-node TCP ingress on ephemeral ports and allows all egress traffic | `bool` | `true` | no | @@ -336,8 +500,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | | [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | -| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | -| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | Cluster IAM role ARN | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | Cluster IAM role name | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | @@ -358,6 +522,9 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | | [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | | [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | +| [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | EKS Auto node IAM role ARN | +| [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | EKS Auto node IAM role name | +| [node\_iam\_role\_unique\_id](#output\_node\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md new file mode 100644 index 0000000000..d12605de93 --- /dev/null +++ b/examples/eks-auto-mode/README.md @@ -0,0 +1,96 @@ +# EKS Auto Mode + +## Usage + +To provision the provided configurations you need to execute: + +```bash +terraform init +terraform plan +terraform apply --auto-approve +``` + +Once the cluster has finished provisioning, you can use the `kubectl` command to interact with the cluster. For example, to deploy a sample deployment and see EKS Auto Mode in action, run: + +```bash +aws eks update-kubeconfig --name $(terraform output -raw cluster_name) +kubectl apply -f deployment.yaml +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [aws](#requirement\_aws) | >= 5.79 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.79 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [disabled\_eks](#module\_disabled\_eks) | ../.. | n/a | +| [eks](#module\_eks) | ../.. | n/a | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | +| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | +| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | +| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | +| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | +| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | Cluster IAM role ARN | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | Cluster IAM role name | +| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | +| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | +| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | +| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | +| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | +| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | +| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | +| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | +| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | +| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | +| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | +| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | +| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | +| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | +| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | +| [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | EKS Auto node IAM role ARN | +| [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | EKS Auto node IAM role name | +| [node\_iam\_role\_unique\_id](#output\_node\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | +| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | +| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | +| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | +| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | +| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | + diff --git a/examples/eks-auto-mode/deployment.yaml b/examples/eks-auto-mode/deployment.yaml new file mode 100644 index 0000000000..a49fc35f4c --- /dev/null +++ b/examples/eks-auto-mode/deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 3 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.10 + resources: + requests: + cpu: 1 diff --git a/examples/eks-auto-mode/main.tf b/examples/eks-auto-mode/main.tf new file mode 100644 index 0000000000..756437d297 --- /dev/null +++ b/examples/eks-auto-mode/main.tf @@ -0,0 +1,86 @@ +provider "aws" { + region = local.region +} + +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + +locals { + name = "ex-${basename(path.cwd)}" + cluster_version = "1.31" + region = "us-west-2" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Test = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true + + enable_cluster_creator_admin_permissions = true + + cluster_compute_config = { + enabled = true + node_pools = ["general-purpose"] + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + tags = local.tags +} + +module "disabled_eks" { + source = "../.." + + create = false +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/tests/fargate-profile/outputs.tf b/examples/eks-auto-mode/outputs.tf similarity index 92% rename from tests/fargate-profile/outputs.tf rename to examples/eks-auto-mode/outputs.tf index 9357464c29..9ed8c27220 100644 --- a/tests/fargate-profile/outputs.tf +++ b/examples/eks-auto-mode/outputs.tf @@ -142,12 +142,12 @@ output "cluster_tls_certificate_sha1_fingerprint" { ################################################################################ output "cluster_iam_role_name" { - description = "IAM role name of the EKS cluster" + description = "Cluster IAM role name" value = module.eks.cluster_iam_role_name } output "cluster_iam_role_arn" { - description = "IAM role ARN of the EKS cluster" + description = "Cluster IAM role ARN" value = module.eks.cluster_iam_role_arn } @@ -156,6 +156,25 @@ output "cluster_iam_role_unique_id" { value = module.eks.cluster_iam_role_unique_id } +################################################################################ +# EKS Auto Node IAM Role +################################################################################ + +output "node_iam_role_name" { + description = "EKS Auto node IAM role name" + value = module.eks.node_iam_role_name +} + +output "node_iam_role_arn" { + description = "EKS Auto node IAM role ARN" + value = module.eks.node_iam_role_arn +} + +output "node_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.node_iam_role_unique_id +} + ################################################################################ # EKS Addons ################################################################################ diff --git a/tests/fargate-profile/variables.tf b/examples/eks-auto-mode/variables.tf similarity index 100% rename from tests/fargate-profile/variables.tf rename to examples/eks-auto-mode/variables.tf diff --git a/tests/fargate-profile/versions.tf b/examples/eks-auto-mode/versions.tf similarity index 82% rename from tests/fargate-profile/versions.tf rename to examples/eks-auto-mode/versions.tf index 0099e6baaf..5b36c201aa 100644 --- a/tests/fargate-profile/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } } } diff --git a/examples/eks-hybrid-nodes/.gitignore b/examples/eks-hybrid-nodes/.gitignore new file mode 100644 index 0000000000..3a7d063a21 --- /dev/null +++ b/examples/eks-hybrid-nodes/.gitignore @@ -0,0 +1,2 @@ +*.pem +*.sh diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md new file mode 100644 index 0000000000..df32f7d8ac --- /dev/null +++ b/examples/eks-hybrid-nodes/README.md @@ -0,0 +1,82 @@ +# EKS Hybrid Node IAM Role + +## Usage + +> [!NOTE] +> The [Packer CLI](https://developer.hashicorp.com/packer/tutorials/docker-get-started/get-started-install-cli) is required to build a custom AMI for the Hybrid node used in the example. + +To provision the provided configurations you need to execute: + +```bash +terraform init +terraform apply -target=module.remote_node_vpc -target=local_file.key_pem --auto-approve +cd ami && packer build -var 'ssh_keypair_name=hybrid-node' -var 'ssh_private_key_file=../key.pem' . && cd - +terraform apply --auto-approve +./join.sh +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [aws](#requirement\_aws) | >= 5.79 | +| [helm](#requirement\_helm) | >= 2.16 | +| [http](#requirement\_http) | >= 3.4 | +| [local](#requirement\_local) | >= 2.5 | +| [tls](#requirement\_tls) | >= 4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.79 | +| [aws.remote](#provider\_aws.remote) | >= 5.79 | +| [helm](#provider\_helm) | >= 2.16 | +| [http](#provider\_http) | >= 3.4 | +| [local](#provider\_local) | >= 2.5 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ../.. | n/a | +| [eks\_hybrid\_node\_role](#module\_eks\_hybrid\_node\_role) | ../../modules/hybrid-node-role | n/a | +| [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | +| [remote\_node\_vpc](#module\_remote\_node\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_instance.hybrid_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance) | resource | +| [aws_route.peer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route) | resource | +| [aws_route.remote_node_private](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route) | resource | +| [aws_route.remote_node_public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route) | resource | +| [aws_security_group.remote_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_ssm_activation.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_activation) | resource | +| [aws_vpc_peering_connection.remote_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_peering_connection) | resource | +| [aws_vpc_peering_connection_accepter.peer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_peering_connection_accepter) | resource | +| [aws_vpc_security_group_egress_rule.remote_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_egress_rule) | resource | +| [aws_vpc_security_group_ingress_rule.remote_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource | +| [helm_release.cilium](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [local_file.join](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.key_pem](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.key_pub_pem](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [aws_ami.hybrid_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_availability_zones.remote](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [http_http.icanhazip](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | + +## Inputs + +No inputs. + +## Outputs + +No outputs. + diff --git a/examples/eks-hybrid-nodes/ami/amazon-eks-ubuntu.pkr.hcl b/examples/eks-hybrid-nodes/ami/amazon-eks-ubuntu.pkr.hcl new file mode 100644 index 0000000000..02c54ea647 --- /dev/null +++ b/examples/eks-hybrid-nodes/ami/amazon-eks-ubuntu.pkr.hcl @@ -0,0 +1,320 @@ +locals { + timestamp = regex_replace(timestamp(), "[- TZ:]", "") + + ami_name = "${var.ami_name_prefix}-${var.eks_version}-amd64-${local.timestamp}" + + tags = { + SourceAMI = "{{ .SourceAMI }}" + Name = local.ami_name + Architecture = "amd64" + } +} + +data "amazon-parameterstore" "this" { + name = "/aws/service/canonical/ubuntu/server-minimal/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id" + region = var.region +} + +################################################################################ +# EBS Source +################################################################################ + +source "amazon-ebs" "this" { + + # AMI Configuration + dynamic "ami_block_device_mappings" { + for_each = var.ami_block_device_mappings + + content { + delete_on_termination = try(ami_block_device_mappings.value.delete_on_termination, true) + device_name = try(ami_block_device_mappings.value.device_name, null) + encrypted = try(ami_block_device_mappings.value.encrypted, null) + iops = try(ami_block_device_mappings.value.iops, null) + no_device = try(ami_block_device_mappings.value.no_device, null) + snapshot_id = try(ami_block_device_mappings.value.snapshot_id, null) + throughput = try(ami_block_device_mappings.value.throughput, null) + virtual_name = try(ami_block_device_mappings.value.virtual_name, null) + volume_size = try(ami_block_device_mappings.value.volume_size, 4) + volume_type = try(ami_block_device_mappings.value.volume_type, "gp3") + kms_key_id = try(ami_block_device_mappings.value.kms_key_id, null) + } + } + + ami_description = var.ami_description + ami_groups = var.ami_groups + ami_name = local.ami_name + ami_org_arns = var.ami_org_arns + ami_ou_arns = var.ami_ou_arns + ami_regions = var.ami_regions + ami_users = var.ami_users + ami_virtualization_type = var.ami_virtualization_type + deprecate_at = var.deprecate_at + ena_support = var.ena_support + encrypt_boot = var.encrypt_boot + force_deregister = var.force_deregister + force_delete_snapshot = var.force_delete_snapshot + imds_support = var.imds_support + kms_key_id = var.kms_key_id + + dynamic "launch_block_device_mappings" { + for_each = length(var.launch_block_device_mappings) > 0 ? var.launch_block_device_mappings : var.ami_block_device_mappings + + content { + delete_on_termination = try(launch_block_device_mappings.value.delete_on_termination, true) + device_name = try(launch_block_device_mappings.value.device_name, null) + encrypted = try(launch_block_device_mappings.value.encrypted, null) + iops = try(launch_block_device_mappings.value.iops, null) + no_device = try(launch_block_device_mappings.value.no_device, null) + snapshot_id = try(launch_block_device_mappings.value.snapshot_id, null) + throughput = try(launch_block_device_mappings.value.throughput, null) + virtual_name = try(launch_block_device_mappings.value.virtual_name, null) + volume_size = try(launch_block_device_mappings.value.volume_size, 4) + volume_type = try(launch_block_device_mappings.value.volume_type, "gp3") + } + } + + region_kms_key_ids = var.region_kms_key_ids + run_volume_tags = var.run_volume_tags + skip_create_ami = var.skip_create_ami + skip_region_validation = var.skip_region_validation + skip_save_build_region = var.skip_save_build_region + sriov_support = var.sriov_support + snapshot_groups = var.snapshot_groups + snapshot_tags = var.snapshot_tags + snapshot_users = var.snapshot_users + tags = merge(local.tags, var.tags) + + # Access Configuration + access_key = var.access_key + + dynamic "assume_role" { + for_each = length(var.assume_role) > 0 ? [var.assume_role] : [] + + content { + duration_seconds = try(assume_role.value.duration_seconds, null) + external_id = try(assume_role.value.external_id, null) + policy = try(assume_role.value.policy, null) + policy_arns = try(assume_role.value.policy_arns, null) + role_arn = try(assume_role.value.role_arn, null) + session_name = try(assume_role.value.session_name, null) + tag = try(assume_role.value.tag, null) + transitive_tag_keys = try(assume_role.value.transitive_tag_keys, null) + } + } + + dynamic "aws_polling" { + for_each = length(var.aws_polling) > 0 ? [var.aws_polling] : [] + + content { + delay_seconds = try(aws_polling.value.delay_seconds, null) + max_attempts = try(aws_polling.value.max_attempts, null) + } + } + + custom_endpoint_ec2 = var.custom_endpoint_ec2 + decode_authorization_messages = var.decode_authorization_messages + insecure_skip_tls_verify = var.insecure_skip_tls_verify + max_retries = var.max_retries + mfa_code = var.mfa_code + profile = var.profile + region = var.region + secret_key = var.secret_key + shared_credentials_file = var.shared_credentials_file + skip_credential_validation = var.skip_credential_validation + skip_metadata_api_check = var.skip_metadata_api_check + token = var.token + + # Communicator + communicator = var.communicator + pause_before_connecting = var.pause_before_connecting + ssh_agent_auth = var.ssh_agent_auth + ssh_bastion_agent_auth = var.ssh_bastion_agent_auth + ssh_bastion_certificate_file = var.ssh_bastion_certificate_file + ssh_bastion_host = var.ssh_bastion_host + ssh_bastion_interactive = var.ssh_bastion_interactive + ssh_bastion_password = var.ssh_bastion_password + ssh_bastion_port = var.ssh_bastion_port + ssh_bastion_private_key_file = var.ssh_bastion_private_key_file + ssh_bastion_username = var.ssh_bastion_username + ssh_ciphers = var.ssh_ciphers + ssh_certificate_file = var.ssh_certificate_file + ssh_clear_authorized_keys = var.ssh_clear_authorized_keys + ssh_disable_agent_forwarding = var.ssh_disable_agent_forwarding + ssh_file_transfer_method = var.ssh_file_transfer_method + ssh_handshake_attempts = var.ssh_handshake_attempts + ssh_host = var.ssh_host + ssh_interface = var.ssh_interface # "public_dns" + ssh_keep_alive_interval = var.ssh_keep_alive_interval + ssh_key_exchange_algorithms = var.ssh_key_exchange_algorithms + ssh_keypair_name = var.ssh_keypair_name + ssh_local_tunnels = var.ssh_local_tunnels + ssh_password = var.ssh_password + ssh_port = var.ssh_port + ssh_private_key_file = var.ssh_private_key_file + ssh_proxy_host = var.ssh_proxy_host + ssh_proxy_password = var.ssh_proxy_password + ssh_proxy_port = var.ssh_proxy_port + ssh_proxy_username = var.ssh_proxy_username + ssh_pty = var.ssh_pty + ssh_read_write_timeout = var.ssh_read_write_timeout + ssh_remote_tunnels = var.ssh_remote_tunnels + ssh_timeout = var.ssh_timeout + ssh_username = var.ssh_username + temporary_key_pair_bits = var.temporary_key_pair_bits + temporary_key_pair_type = var.temporary_key_pair_type + + # Run Configuration + associate_public_ip_address = var.associate_public_ip_address + capacity_reservation_preference = var.capacity_reservation_preference + capacity_reservation_group_arn = var.capacity_reservation_group_arn + capacity_reservation_id = var.capacity_reservation_id + disable_stop_instance = var.disable_stop_instance + ebs_optimized = var.ebs_optimized + enable_nitro_enclave = var.enable_nitro_enclave + enable_unlimited_credits = var.enable_unlimited_credits + iam_instance_profile = var.iam_instance_profile + instance_type = var.instance_type + fleet_tags = var.fleet_tags + pause_before_ssm = var.pause_before_ssm + + dynamic "placement" { + for_each = length(var.placement) > 0 ? [var.placement] : [] + + content { + host_resource_group_arn = try(placement.value.host_resource_group_arn, null) + tenancy = try(placement.value.tenancy, null) + } + } + + run_tags = merge(local.tags, var.run_tags) + security_group_ids = var.security_group_ids + + dynamic "security_group_filter" { + for_each = length(var.security_group_filter) > 0 ? var.security_group_filter : [] + + content { + filters = try(security_group_filter.value.filters, null) + } + } + + session_manager_port = var.session_manager_port + shutdown_behavior = var.shutdown_behavior + skip_profile_validation = var.skip_profile_validation + source_ami = data.amazon-parameterstore.this.value + + dynamic "subnet_filter" { + for_each = length(var.subnet_filter) > 0 ? [var.subnet_filter] : [] + + content { + filters = try(subnet_filter.value.filters, null) + most_free = try(subnet_filter.value.most_free, null) + random = try(subnet_filter.value.random, null) + } + } + + subnet_id = var.subnet_id + + dynamic "temporary_iam_instance_profile_policy_document" { + for_each = length(var.temporary_iam_instance_profile_policy_document) > 0 ? [var.temporary_iam_instance_profile_policy_document] : [] + + content { + dynamic "Statement" { + for_each = temporary_iam_instance_profile_policy_document.value + + content { + Action = try(Statement.value.Action, []) + Effect = try(Statement.value.Effect, "Allow") + Resource = try(Statement.value.Resource, ["*"]) + } + } + Version = "2012-10-17" + } + } + + temporary_security_group_source_cidrs = var.temporary_security_group_source_cidrs + temporary_security_group_source_public_ip = var.temporary_security_group_source_public_ip + user_data = var.user_data + user_data_file = var.user_data_file + + dynamic "vpc_filter" { + for_each = length(var.vpc_filter) > 0 ? var.vpc_filter : [] + + content { + filters = try(vpc_filter.value.filters, null) + } + } + + vpc_id = var.vpc_id + + dynamic "metadata_options" { + for_each = length(var.metadata_options) > 0 ? [var.metadata_options] : [] + + content { + http_endpoint = try(metadata_options.value.http_endpoint, null) + http_put_response_hop_limit = try(metadata_options.value.http_put_response_hop_limit, null) + http_tokens = try(metadata_options.value.http_tokens, null) + instance_metadata_tags = try(metadata_options.value.instance_metadata_tags, null) + } + } +} + +################################################################################ +# Build +################################################################################ + +build { + sources = ["source.amazon-ebs.this"] + + provisioner "shell" { + execute_command = "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'" + + env = { + DEBIAN_FRONTEND = "noninteractive" + } + + expect_disconnect = true + + inline = [ + "cloud-init status --wait", + "apt update", + "apt upgrade -y", + "apt install iptables conntrack -y", + "systemctl reboot", + ] + + pause_after = "15s" + } + + provisioner "shell" { + execute_command = "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'" + + env = { + DEBIAN_FRONTEND = "noninteractive" + } + + inline = [ + + "snap install aws-cli --classic", + "snap switch --channel=candidate amazon-ssm-agent", + "curl -OL 'https://hybrid-assets.eks.amazonaws.com/releases/latest/bin/linux/amd64/nodeadm'", + "mv nodeadm /usr/bin/nodeadm", + "chmod +x /usr/bin/nodeadm", + "nodeadm install ${var.eks_version} --credential-provider ${var.credential_provider}", + ] + } + + provisioner "shell" { + execute_command = "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'" + + env = { + DEBIAN_FRONTEND = "noninteractive" + } + + inline = [ + "apt --purge autoremove -y", + "cloud-init clean --logs --machine-id", + "mkdir -p /etc/amazon/ssm", + "cp $(find / -name '*seelog.xml.template') /etc/amazon/ssm/seelog.xml", + ] + } +} diff --git a/examples/eks-hybrid-nodes/ami/plugins.pkr.hcl b/examples/eks-hybrid-nodes/ami/plugins.pkr.hcl new file mode 100644 index 0000000000..24a7f5b8f5 --- /dev/null +++ b/examples/eks-hybrid-nodes/ami/plugins.pkr.hcl @@ -0,0 +1,8 @@ +packer { + required_plugins { + amazon = { + version = "~> 1.2" + source = "github.com/hashicorp/amazon" + } + } +} diff --git a/examples/eks-hybrid-nodes/ami/variables.pkr.hcl b/examples/eks-hybrid-nodes/ami/variables.pkr.hcl new file mode 100644 index 0000000000..89e1b27ffa --- /dev/null +++ b/examples/eks-hybrid-nodes/ami/variables.pkr.hcl @@ -0,0 +1,723 @@ +variable "ami_name_prefix" { + description = "The prefix to use when creating the AMI name. i.e. - `---" + type = string + default = "eks-hybrid-ubuntu" +} + +variable "eks_version" { + description = "The EKS cluster version associated with the AMI created" + type = string + default = "1.31" +} + +variable "credential_provider" { + description = "The credential provider to use with the Hybrid Node role" + type = string + default = "ssm" +} + +variable "cpu_architecture" { + description = "The CPU architecture. Either `amd64` or `arm64`" + type = string + default = "amd64" +} + +################################################################################ +# EBS Source +################################################################################ + +variable "ami_block_device_mappings" { + description = "The block device mappings attached when booting a new instance from the AMI created" + type = list(map(string)) + default = [ + { + device_name = "/dev/sda1" + volume_size = 24 + volume_type = "gp3" + delete_on_termination = true + }, + ] +} + +variable "ami_description" { + description = "The description to use when creating the AMI" + type = string + default = "EKS Hybrid Node demonstration AMI" +} + +variable "ami_groups" { + description = "A list of groups that have access to launch the resulting AMI(s). By default no groups have permission to launch the AMI. `all` will make the AMI publicly accessible. AWS currently doesn't accept any value other than `all`" + type = list(string) + default = null +} + +variable "ami_org_arns" { + description = "A list of Amazon Resource Names (ARN) of AWS Organizations that have access to launch the resulting AMI(s). By default no organizations have permission to launch the AMI" + type = list(string) + default = null +} + +variable "ami_ou_arns" { + description = "A list of Amazon Resource Names (ARN) of AWS Organizations organizational units (OU) that have access to launch the resulting AMI(s). By default no organizational units have permission to launch the AMI" + type = list(string) + default = null +} + +variable "ami_regions" { + description = "A list of regions to copy the AMI to. Tags and attributes are copied along with the AMI. AMI copying takes time depending on the size of the AMI, but will generally take many minutes" + type = list(string) + default = null +} + +variable "ami_users" { + description = "A list of account IDs that have access to launch the resulting AMI(s). By default no additional users other than the user creating the AMI has permissions to launch it" + type = list(string) + default = null +} + +variable "ami_virtualization_type" { + description = "The type of virtualization used to create the AMI. Can be one of `hvm` or `paravirtual`" + type = string + default = "hvm" +} + +variable "deprecate_at" { + description = "The date and time to deprecate the AMI, in UTC, in the following format: YYYY-MM-DDTHH:MM:SSZ. If you specify a value for seconds, Amazon EC2 rounds the seconds to the nearest minute" + type = string + default = null +} + +variable "ena_support" { + description = "Enable enhanced networking (ENA but not SriovNetSupport) on HVM-compatible AMIs" + type = bool + default = null +} + +variable "encrypt_boot" { + description = "Whether or not to encrypt the resulting AMI when copying a provisioned instance to an AMI. By default, Packer will keep the encryption setting to what it was in the source image" + type = bool + default = null +} + +variable "force_deregister" { + description = "Force Packer to first deregister an existing AMI if one with the same name already exists. Default `false`" + type = bool + default = null +} + +variable "force_delete_snapshot" { + description = "Force Packer to delete snapshots associated with AMIs, which have been deregistered by force_deregister. Default `false`" + type = bool + default = null +} + +variable "imds_support" { + description = "Enforce version of the Instance Metadata Service on the built AMI. Valid options are `unset` (legacy) and `v2.0`" + type = string + default = "v2.0" +} + +variable "kms_key_id" { + description = "ID, alias or ARN of the KMS key to use for AMI encryption. This only applies to the main `region` -- any regions the AMI gets copied to copied will be encrypted by the default EBS KMS key for that region, unless you set region-specific keys in `region_kms_key_ids`" + type = string + default = null +} + +variable "launch_block_device_mappings" { + description = "The block device mappings to use when creating the AMI. If you add instance store volumes or EBS volumes in addition to the root device volume, the created AMI will contain block device mapping information for those volumes. Amazon creates snapshots of the source instance's root volume and any other EBS volumes described here. When you launch an instance from this new AMI, the instance automatically launches with these additional volumes, and will restore them from snapshots taken from the source instance" + type = list(map(string)) + default = [ + { + device_name = "/dev/sda1" + volume_size = 24 + volume_type = "gp3" + delete_on_termination = true + }, + ] +} + +variable "region_kms_key_ids" { + description = "regions to copy the ami to, along with the custom kms key id (alias or arn) to use for encryption for that region. Keys must match the regions provided in `ami_regions`" + type = map(string) + default = null +} + +variable "run_volume_tags" { + description = "Tags to apply to the volumes that are launched to create the AMI. These tags are not applied to the resulting AMI" + type = map(string) + default = null +} + +variable "skip_create_ami" { + description = "If `true`, Packer will not create the AMI. Useful for setting to `true` during a build test stage. Default `false`" + type = bool + default = null +} + +variable "skip_region_validation" { + description = "Set to `true` if you want to skip validation of the `ami_regions` configuration option. Default `false`" + type = bool + default = null +} + +variable "skip_save_build_region" { + description = "If true, Packer will not check whether an AMI with the ami_name exists in the region it is building in. It will use an intermediary AMI name, which it will not convert to an AMI in the build region. Default `false`" + type = bool + default = null +} + +variable "sriov_support" { + description = "Enable enhanced networking (SriovNetSupport but not ENA) on HVM-compatible AMIs" + type = bool + default = null +} + +variable "snapshot_groups" { + description = "A list of groups that have access to create volumes from the snapshot(s). By default no groups have permission to create volumes from the snapshot(s). all will make the snapshot publicly accessible" + type = list(string) + default = null +} + +variable "snapshot_tags" { + description = "Key/value pair tags to apply to snapshot. They will override AMI tags if already applied to snapshot" + type = map(string) + default = null +} + +variable "snapshot_users" { + description = "A list of account IDs that have access to create volumes from the snapshot(s). By default no additional users other than the user creating the AMI has permissions to create volumes from the backing snapshot(s)" + type = list(string) + default = null +} + +variable "tags" { + description = "Key/value pair tags applied to the AMI" + type = map(string) + default = {} +} + +# Access Configuration + +variable "access_key" { + description = "The access key used to communicate with AWS" + type = string + default = null +} + +variable "assume_role" { + description = "If provided with a role ARN, Packer will attempt to assume this role using the supplied credentials" + type = map(string) + default = {} +} + +variable "aws_polling" { + description = "Polling configuration for the AWS waiter. Configures the waiter for resources creation or actions like attaching volumes or importing image" + type = map(string) + default = {} +} + +variable "custom_endpoint_ec2" { + description = "This option is useful if you use a cloud provider whose API is compatible with aws EC2" + type = string + default = null +} + +variable "decode_authorization_messages" { + description = "Enable automatic decoding of any encoded authorization (error) messages using the sts:DecodeAuthorizationMessage API" + type = bool + default = null +} + +variable "insecure_skip_tls_verify" { + description = "This allows skipping TLS verification of the AWS EC2 endpoint. The default is `false`" + type = bool + default = null +} + +variable "max_retries" { + description = "This is the maximum number of times an API call is retried, in the case where requests are being throttled or experiencing transient failures. The delay between the subsequent API calls increases exponentially" + type = number + default = null +} + +variable "mfa_code" { + description = "The MFA TOTP code. This should probably be a user variable since it changes all the time" + type = string + default = null +} + +variable "profile" { + description = "The profile to use in the shared credentials file for AWS" + type = string + default = null +} + +variable "region" { + description = "The name of the region, such as us-east-1, in which to launch the EC2 instance to create the AMI" + type = string + default = "us-east-1" +} + +variable "secret_key" { + description = "The secret key used to communicate with AWS" + type = string + default = null +} + +variable "shared_credentials_file" { + description = "Path to a credentials file to load credentials from" + type = string + default = null +} + +variable "skip_credential_validation" { + description = "Set to true if you want to skip validating AWS credentials before runtime" + type = bool + default = null +} + +variable "skip_metadata_api_check" { + description = "Skip Metadata Api Check" + type = bool + default = null +} + +variable "token" { + description = "The access token to use. This is different from the access key and secret key" + type = string + default = null +} + +# Communicator + +variable "communicator" { + description = "The communicator to use to communicate with the EC2 instance. Valid values are `none`, `ssh`, `winrm`, and `ssh+winrm`" + type = string + default = "ssh" +} + +variable "pause_before_connecting" { + description = "We recommend that you enable SSH or WinRM as the very last step in your guest's bootstrap script, but sometimes you may have a race condition where you need Packer to wait before attempting to connect to your guest" + type = string + default = null +} + +variable "ssh_agent_auth" { + description = "If true, the local SSH agent will be used to authenticate connections to the source instance. No temporary keypair will be created, and the values of `ssh_password` and `ssh_private_key_file` will be ignored. The environment variable `SSH_AUTH_SOCK` must be set for this option to work properly" + type = bool + default = null +} + +variable "ssh_bastion_agent_auth" { + description = "If `true`, the local SSH agent will be used to authenticate with the bastion host. Defaults to `false`" + type = bool + default = null +} + +variable "ssh_bastion_certificate_file" { + description = "Path to user certificate used to authenticate with bastion host. The ~ can be used in path and will be expanded to the home directory of current user" + type = string + default = null +} + +variable "ssh_bastion_host" { + description = "A bastion host to use for the actual SSH connection" + type = string + default = null +} + +variable "ssh_bastion_interactive" { + description = "If `true`, the keyboard-interactive used to authenticate with bastion host" + type = bool + default = null +} + +variable "ssh_bastion_password" { + description = "The password to use to authenticate with the bastion host" + type = string + default = null +} + +variable "ssh_bastion_port" { + description = "The port of the bastion host. Defaults to `22`" + type = number + default = null +} + +variable "ssh_bastion_private_key_file" { + description = "Path to a PEM encoded private key file to use to authenticate with the bastion host. The `~` can be used in path and will be expanded to the home directory of current user" + type = string + default = null +} + +variable "ssh_bastion_username" { + description = "The username to connect to the bastion host" + type = string + default = null +} + +variable "ssh_ciphers" { + description = "This overrides the value of ciphers supported by default by Golang. The default value is `[\"aes128-gcm@openssh.com\", \"chacha20-poly1305@openssh.com\", \"aes128-ctr\", \"aes192-ctr\", \"aes256-ctr\"]`" + type = list(string) + default = null +} + +variable "ssh_certificate_file" { + description = "Path to user certificate used to authenticate with SSH. The `~` can be used in path and will be expanded to the home directory of current user" + type = string + default = null +} + +variable "ssh_clear_authorized_keys" { + description = "If true, Packer will attempt to remove its temporary key from `~/.ssh/authorized_keys` and `/root/.ssh/authorized_keys`" + type = bool + default = null +} + +variable "ssh_disable_agent_forwarding" { + description = "If `true`, SSH agent forwarding will be disabled. Defaults to `false`" + type = bool + default = null +} + +variable "ssh_file_transfer_method" { + description = "How to transfer files, Secure copy (`scp` default) or SSH File Transfer Protocol (`sftp`)" + type = string + default = null +} + +variable "ssh_handshake_attempts" { + description = "The number of handshakes to attempt with SSH once it can connect. This defaults to `10`, unless a `ssh_timeout` is set" + type = number + default = null +} + +variable "ssh_host" { + description = "The address to SSH to. This usually is automatically configured by the builder" + type = string + default = null +} + +variable "ssh_interface" { + description = "One of `public_ip`, `private_ip`, `public_dns`, `private_dns` or `session_manager`. If set, either the public IP address, private IP address, public DNS name or private DNS name will be used as the host for SSH. The default behavior if inside a VPC is to use the public IP address if available, otherwise the private IP address will be used. If not in a VPC the public DNS name will be used" + type = string + default = "public_ip" +} + +variable "ssh_keep_alive_interval" { + description = "How often to send \"keep alive\" messages to the server. Set to a negative value (`-1s`) to disable. Defaults to `5s`" + type = string + default = null +} + +variable "ssh_key_exchange_algorithms" { + description = "If set, Packer will override the value of key exchange (kex) algorithms supported by default by Golang. Acceptable values include: `curve25519-sha256@libssh.org`, `ecdh-sha2-nistp256`, `ecdh-sha2-nistp384`, `ecdh-sha2-nistp521`, `diffie-hellman-group14-sha1`, and `diffie-hellman-group1-sha1`" + type = list(string) + default = null +} + +variable "ssh_keypair_name" { + description = "If specified, this is the key that will be used for SSH with the machine. The key must match a key pair name loaded up into the remote" + type = string + default = null +} + +variable "ssh_local_tunnels" { + description = "A list of local tunnels to use when connecting to the host" + type = list(string) + default = null +} + +variable "ssh_password" { + description = "A plaintext password to use to authenticate with SSH" + type = string + default = null +} + +variable "ssh_port" { + description = "The port to connect to SSH. This defaults to `22`" + type = number + default = null +} + +variable "ssh_private_key_file" { + description = "Path to a PEM encoded private key file to use to authenticate with SSH. The ~ can be used in path and will be expanded to the home directory of current user" + type = string + default = null +} + +variable "ssh_proxy_host" { + description = "A SOCKS proxy host to use for SSH connection" + type = string + default = null +} + +variable "ssh_proxy_password" { + description = "The optional password to use to authenticate with the proxy server" + type = string + default = null +} + +variable "ssh_proxy_port" { + description = "A port of the SOCKS proxy. Defaults to `1080`" + type = number + default = null +} + +variable "ssh_proxy_username" { + description = "The optional username to authenticate with the proxy server" + type = string + default = null +} + +variable "ssh_pty" { + description = "If `true`, a PTY will be requested for the SSH connection. This defaults to `false`" + type = bool + default = null +} + +variable "ssh_read_write_timeout" { + description = "The amount of time to wait for a remote command to end. This might be useful if, for example, packer hangs on a connection after a reboot. Example: `5m`. Disabled by default" + type = string + default = null +} + +variable "ssh_remote_tunnels" { + description = "A list of remote tunnels to use when connecting to the host" + type = list(string) + default = null +} + +variable "ssh_timeout" { + description = "The time to wait for SSH to become available. Packer uses this to determine when the machine has booted so this is usually quite long. This defaults to `5m`, unless `ssh_handshake_attempts` is set" + type = string + default = null +} + +variable "ssh_username" { + description = "The username to connect to SSH with. Required if using SSH" + type = string + default = "ubuntu" +} + +variable "temporary_key_pair_type" { + description = "Specifies the type of key to create. The possible values are 'dsa', 'ecdsa', 'ed25519', or 'rsa'. Default is `ed25519`" + type = string + default = "ed25519" +} + +variable "temporary_key_pair_bits" { + description = "Specifies the number of bits in the key to create. For RSA keys, the minimum size is 1024 bits and the default is 4096 bits. Generally, 3072 bits is considered sufficient" + type = number + default = null +} + +# Run Configuration + +variable "associate_public_ip_address" { + description = "If using a non-default VPC, public IP addresses are not provided by default. If this is true, your new instance will get a Public IP" + type = bool + default = true +} + +variable "capacity_reservation_preference" { + description = "Set the preference for using a capacity reservation if one exists. Either will be `open` or `none`. Defaults to `none`" + type = string + default = null +} + +variable "capacity_reservation_group_arn" { + description = "Provide the EC2 Capacity Reservation Group ARN that will be used by Packer" + type = string + default = null +} + +variable "capacity_reservation_id" { + description = "Provide the specific EC2 Capacity Reservation ID that will be used by Packer" + type = string + default = null +} + +variable "disable_stop_instance" { + description = "If this is set to true, Packer will not stop the instance but will assume that you will send the stop signal yourself through your final provisioner" + type = bool + default = null +} + +variable "ebs_optimized" { + description = "Mark instance as EBS Optimized. Default `false`" + type = bool + default = null +} + +variable "enable_nitro_enclave" { + description = "Enable support for Nitro Enclaves on the instance" + type = bool + default = null +} + +variable "enable_unlimited_credits" { + description = "Enabling Unlimited credits allows the source instance to burst additional CPU beyond its available CPU Credits for as long as the demand exists" + type = bool + default = null +} + +variable "iam_instance_profile" { + description = "The name of an IAM instance profile to launch the EC2 instance with" + type = string + default = null +} + +variable "instance_type" { + description = "The EC2 instance type to use while building the AMI, such as `m5.large`" + type = string + default = "c5.xlarge" +} + +variable "fleet_tags" { + description = "Key/value pair tags to apply tags to the fleet that is issued" + type = map(string) + default = null +} + +variable "pause_before_ssm" { + description = "The time to wait before establishing the Session Manager session" + type = string + default = null +} + +variable "placement" { + description = "Describes the placement of an instance" + type = map(string) + default = {} +} + +variable "run_tags" { + description = "Key/value pair tags to apply to the generated key-pair, security group, iam profile and role, snapshot, network interfaces and instance that is launched to create the EBS volumes. The resulting AMI will also inherit these tags" + type = map(string) + default = null +} + +variable "security_group_ids" { + description = "A list of security group IDs to assign to the instance. By default this is not set and Packer will automatically create a new temporary security group to allow SSH access" + type = list(string) + default = null +} + +variable "security_group_filter" { + description = "Filters used to populate the `security_group_ids` field. `security_group_ids` take precedence over this" + type = list(map(string)) + default = [] +} + +variable "session_manager_port" { + description = "Which port to connect the local end of the session tunnel to. If left blank, Packer will choose a port for you from available ports. This option is only used when `ssh_interface` is set `session_manager`" + type = number + default = null +} + +variable "shutdown_behavior" { + description = "Automatically terminate instances on shutdown in case Packer exits ungracefully. Possible values are `stop` and `terminate`. Defaults to `stop`" + type = string + default = null +} + +variable "skip_profile_validation" { + description = "Whether or not to check if the IAM instance profile exists. Defaults to `false`" + type = bool + default = null +} + +variable "subnet_filter" { + description = "Filters used to populate the subnet_id field. `subnet_id` take precedence over this" + default = { + filters = { + "tag:eks-hybrid-packer" = "true" + } + random = true + } +} + +variable "subnet_id" { + description = "f using VPC, the ID of the subnet, such as subnet-12345def, where Packer will launch the EC2 instance. This field is required if you are using an non-default VPC" + type = string + default = null +} + +variable "temporary_iam_instance_profile_policy_document" { + description = "Creates a temporary instance profile policy document to grant permissions to the EC2 instance. This is an alternative to using an existing `iam_instance_profile`" + default = [ + { + Effect = "Allow" + Action = [ + "ec2:Describe*", + ] + Resource = ["*"] + }, + ] +} + +variable "temporary_security_group_source_cidrs" { + description = "A list of IPv4 CIDR blocks to be authorized access to the instance, when packer is creating a temporary security group. The default is `[0.0.0.0/0]`" + type = list(string) + default = null +} + +variable "temporary_security_group_source_public_ip" { + description = "When enabled, use public IP of the host (obtained from https://checkip.amazonaws.com) as CIDR block to be authorized access to the instance, when packer is creating a temporary security group. Defaults to `false`" + type = bool + default = null +} + +variable "user_data" { + description = "User data to apply when launching the instance" + type = string + default = null +} + +variable "user_data_file" { + description = "Path to a file that will be used for the user data when launching the instance" + type = string + default = null +} + +variable "vpc_filter" { + description = "Filters used to populate the `vpc_id` field. `vpc_id` take precedence over this" + type = list(map(string)) + default = [] +} + +variable "vpc_id" { + description = "If launching into a VPC subnet, Packer needs the VPC ID in order to create a temporary security group within the VPC. Requires `subnet_id` to be set. If this field is left blank, Packer will try to get the VPC ID from the `subnet_id`" + type = string + default = null +} + +variable "metadata_options" { + description = "Configures the metadata options for the instance launched" + type = map(string) + default = { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + } +} + +################################################################################ +# Build +################################################################################ + +variable "shell_provisioner1" { + description = "Values passed to the first shell provisioner" + default = {} +} + +variable "shell_provisioner2" { + description = "Values passed to the second shell provisioner" + default = {} +} + +variable "shell_provisioner3" { + description = "Values passed to the third/last shell provisioner" + default = {} +} diff --git a/examples/eks-hybrid-nodes/main.tf b/examples/eks-hybrid-nodes/main.tf new file mode 100644 index 0000000000..8791e6cb53 --- /dev/null +++ b/examples/eks-hybrid-nodes/main.tf @@ -0,0 +1,148 @@ +provider "aws" { + region = local.region +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } + } +} + +locals { + name = "ex-${basename(path.cwd)}" + region = "us-west-2" + + cluster_version = "1.31" + + tags = { + Test = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# EKS Cluster +################################################################################ + +module "eks" { + source = "../.." + + cluster_name = local.name + cluster_version = local.cluster_version + + cluster_endpoint_public_access = true + enable_cluster_creator_admin_permissions = true + + cluster_addons = { + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + } + + create_node_security_group = false + cluster_security_group_additional_rules = { + hybrid-all = { + cidr_blocks = [local.remote_network_cidr] + description = "Allow all traffic from remote node/pod network" + from_port = 0 + to_port = 0 + protocol = "all" + type = "ingress" + } + } + + cluster_compute_config = { + enabled = true + node_pools = ["system"] + } + + access_entries = { + hybrid-node-role = { + principal_arn = module.eks_hybrid_node_role.arn + type = "HYBRID_LINUX" + } + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + cluster_remote_network_config = { + remote_node_networks = { + cidrs = [local.remote_node_cidr] + } + remote_pod_networks = { + cidrs = [local.remote_pod_cidr] + } + } + + tags = local.tags +} + +################################################################################ +# VPC +################################################################################ + +locals { + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) +} + +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} + +################################################################################ +# VPC Peering Connection +################################################################################ + +resource "aws_vpc_peering_connection_accepter" "peer" { + vpc_peering_connection_id = aws_vpc_peering_connection.remote_node.id + auto_accept = true + + tags = local.tags +} + +resource "aws_route" "peer" { + route_table_id = one(module.vpc.private_route_table_ids) + destination_cidr_block = local.remote_network_cidr + vpc_peering_connection_id = aws_vpc_peering_connection.remote_node.id +} diff --git a/examples/eks-hybrid-nodes/outputs.tf b/examples/eks-hybrid-nodes/outputs.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/eks-hybrid-nodes/remote.tf b/examples/eks-hybrid-nodes/remote.tf new file mode 100644 index 0000000000..3a44aba523 --- /dev/null +++ b/examples/eks-hybrid-nodes/remote.tf @@ -0,0 +1,314 @@ +provider "aws" { + alias = "remote" + region = "us-east-1" +} + +################################################################################ +# Hybrid Node IAM Module +################################################################################ + +module "eks_hybrid_node_role" { + source = "../../modules/hybrid-node-role" + + tags = local.tags +} + +################################################################################ +# Psuedo Hybrid Node +# Demonstration only - AWS EC2 instances are not supported for EKS Hybrid nodes +################################################################################ + +# Activation should be done is same region as cluster +resource "aws_ssm_activation" "this" { + name = "hybrid-node" + iam_role = module.eks_hybrid_node_role.name + registration_limit = 10 + + tags = local.tags +} + +module "key_pair" { + source = "terraform-aws-modules/key-pair/aws" + version = "~> 2.0" + + providers = { + aws = aws.remote + } + + key_name = "hybrid-node" + create_private_key = true + + tags = local.tags +} + +resource "local_file" "key_pem" { + content = module.key_pair.private_key_pem + filename = "key.pem" + file_permission = "0600" +} + +resource "local_file" "key_pub_pem" { + content = module.key_pair.public_key_pem + filename = "key_pub.pem" + file_permission = "0600" +} + +resource "local_file" "join" { + content = <<-EOT + #!/usr/bin/env bash + + cat < nodeConfig.yaml + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + cluster: + name: ${module.eks.cluster_name} + region: ${local.region} + hybrid: + ssm: + activationCode: ${aws_ssm_activation.this.activation_code} + activationId: ${aws_ssm_activation.this.id} + EOF + + # Use SCP/SSH to execute commands on the remote host + scp -i ${local_file.key_pem.filename} nodeConfig.yaml ubuntu@${aws_instance.hybrid_node["one"].public_ip}:/home/ubuntu/nodeConfig.yaml + ssh -n -i ${local_file.key_pem.filename} ubuntu@${aws_instance.hybrid_node["one"].public_ip} sudo nodeadm init -c file://nodeConfig.yaml + ssh -n -i ${local_file.key_pem.filename} ubuntu@${aws_instance.hybrid_node["one"].public_ip} sudo systemctl daemon-reload + + scp -i ${local_file.key_pem.filename} nodeConfig.yaml ubuntu@${aws_instance.hybrid_node["two"].public_ip}:/home/ubuntu/nodeConfig.yaml + ssh -n -i ${local_file.key_pem.filename} ubuntu@${aws_instance.hybrid_node["two"].public_ip} sudo nodeadm init -c file://nodeConfig.yaml + ssh -n -i ${local_file.key_pem.filename} ubuntu@${aws_instance.hybrid_node["two"].public_ip} sudo systemctl daemon-reload + + # Clean up + rm nodeConfig.yaml + EOT + filename = "join.sh" +} + +data "aws_ami" "hybrid_node" { + provider = aws.remote + + most_recent = true + name_regex = "eks-hybrid-ubuntu-${local.cluster_version}-amd64-*" + owners = ["self"] +} + +# Demonstration only - AWS EC2 instances are not supported for EKS Hybrid nodes +resource "aws_instance" "hybrid_node" { + provider = aws.remote + + for_each = { one = 0, two = 1 } + + ami = data.aws_ami.hybrid_node.id + associate_public_ip_address = true + instance_type = "m5.large" + + # Block IMDS to make instance look less like EC2 and more like vanilla VM + metadata_options { + http_endpoint = "disabled" + } + + vpc_security_group_ids = [aws_security_group.remote_node.id] + subnet_id = element(module.remote_node_vpc.public_subnets, each.value) + + tags = merge( + local.tags, + { Name = "hybrid-node-${each.key}" } + ) +} + +################################################################################ +# Psuedo Hybrid Node - Security Group +# Demonstration only - AWS EC2 instances are not supported for EKS Hybrid nodes +################################################################################ + +# Retrieve the IP of where the Terraform is running to restrict SSH access to that IP +data "http" "icanhazip" { + url = "http://icanhazip.com" +} + +resource "aws_security_group" "remote_node" { + provider = aws.remote + + name = "hybrid-node" + vpc_id = module.remote_node_vpc.vpc_id + revoke_rules_on_delete = true + + tags = merge( + local.tags, + { Name = "hybrid-node" } + ) +} + +resource "aws_vpc_security_group_ingress_rule" "remote_node" { + provider = aws.remote + + for_each = { + cluster-all = { + description = "Allow all traffic from cluster network" + cidr_ipv4 = module.vpc.vpc_cidr_block + ip_protocol = "all" + } + remote-all = { + description = "Allow all traffic from within the remote network itself" + ip_protocol = "all" + referenced_security_group_id = aws_security_group.remote_node.id + } + # Restrict SSH access to only the IP where Terraform is running + ssh = { + description = "Local SSH access to join node to cluster" + cidr_ipv4 = "${chomp(data.http.icanhazip.response_body)}/32" + from_port = "22" + ip_protocol = "tcp" + } + } + + cidr_ipv4 = try(each.value.cidr_ipv4, null) + from_port = try(each.value.from_port, null) + ip_protocol = try(each.value.ip_protocol, null) + to_port = try(each.value.to_port, each.value.from_port, null) + referenced_security_group_id = try(each.value.referenced_security_group_id, null) + security_group_id = aws_security_group.remote_node.id + + tags = merge( + local.tags, + { Name = "hybrid-node-${each.key}" } + ) +} + +resource "aws_vpc_security_group_egress_rule" "remote_node" { + provider = aws.remote + + for_each = { + all = { + description = "Allow all egress" + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "all" + } + } + + cidr_ipv4 = try(each.value.cidr_ipv4, null) + from_port = try(each.value.from_port, null) + ip_protocol = try(each.value.ip_protocol, null) + to_port = try(each.value.to_port, each.value.from_port, null) + referenced_security_group_id = try(each.value.referenced_security_group_id, null) + security_group_id = aws_security_group.remote_node.id + + tags = merge( + local.tags, + { Name = "hybrid-node-${each.key}" } + ) +} + +################################################################################ +# Cilium CNI +################################################################################ + +resource "helm_release" "cilium" { + name = "cilium" + repository = "https://helm.cilium.io/" + chart = "cilium" + version = "1.16.4" + namespace = "kube-system" + wait = false + + values = [ + <<-EOT + nodeSelector: + eks.amazonaws.com/compute-type: hybrid + ipam: + mode: cluster-pool + operator: + clusterPoolIPv4MaskSize: 26 + clusterPoolIPv4PodCIDRList: + - ${local.remote_pod_cidr} + operator: + unmanagedPodWatcher: + restart: false + EOT + ] +} + +################################################################################ +# VPC +################################################################################ + +locals { + remote_network_cidr = "172.16.0.0/16" + remote_node_cidr = cidrsubnet(local.remote_network_cidr, 2, 0) + remote_pod_cidr = cidrsubnet(local.remote_network_cidr, 2, 1) + + remote_node_azs = slice(data.aws_availability_zones.remote.names, 0, 3) +} + +data "aws_availability_zones" "remote" { + provider = aws.remote + + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + +module "remote_node_vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + providers = { + aws = aws.remote + } + + name = local.name + cidr = local.remote_network_cidr + + azs = local.remote_node_azs + private_subnets = [for k, v in local.remote_node_azs : cidrsubnet(local.remote_network_cidr, 4, k)] + public_subnets = [for k, v in local.remote_node_azs : cidrsubnet(local.remote_network_cidr, 8, k + 48)] + + public_subnet_tags = { + # For building the AMI + "eks-hybrid-packer" : "true" + } + + enable_nat_gateway = true + single_nat_gateway = true + + tags = local.tags +} + +################################################################################ +# VPC Peering Connection +################################################################################ + +resource "aws_vpc_peering_connection" "remote_node" { + provider = aws.remote + + auto_accept = false + + peer_vpc_id = module.vpc.vpc_id + peer_region = local.region + + vpc_id = module.remote_node_vpc.vpc_id + + tags = merge( + local.tags, + { Name = "remote-node" } + ) +} + +resource "aws_route" "remote_node_private" { + provider = aws.remote + + route_table_id = one(module.remote_node_vpc.private_route_table_ids) + destination_cidr_block = module.vpc.vpc_cidr_block + vpc_peering_connection_id = aws_vpc_peering_connection.remote_node.id +} + +resource "aws_route" "remote_node_public" { + provider = aws.remote + + route_table_id = one(module.remote_node_vpc.public_route_table_ids) + destination_cidr_block = module.vpc.vpc_cidr_block + vpc_peering_connection_id = aws_vpc_peering_connection.remote_node.id +} diff --git a/examples/eks-hybrid-nodes/variables.tf b/examples/eks-hybrid-nodes/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf new file mode 100644 index 0000000000..6cd8533c17 --- /dev/null +++ b/examples/eks-hybrid-nodes/versions.tf @@ -0,0 +1,26 @@ +terraform { + required_version = ">= 1.3.2" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.79" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.16" + } + http = { + source = "hashicorp/http" + version = ">= 3.4" + } + local = { + source = "hashicorp/local" + version = ">= 2.5" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } + } +} diff --git a/examples/eks-managed-node-group/main.tf b/examples/eks-managed-node-group/main.tf index 4409a2e388..451bbdb385 100644 --- a/examples/eks-managed-node-group/main.tf +++ b/examples/eks-managed-node-group/main.tf @@ -2,7 +2,13 @@ provider "aws" { region = local.region } -data "aws_availability_zones" "available" {} +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} locals { name = "ex-eks-mng" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index 0099e6baaf..5b36c201aa 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 15d51bcdb9..87a7d8ae78 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -89,7 +89,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.75 | +| [aws](#requirement\_aws) | >= 5.79 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -97,8 +97,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.75 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.75 | +| [aws](#provider\_aws) | >= 5.79 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.79 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | @@ -128,52 +128,5 @@ No inputs. ## Outputs -| Name | Description | -|------|-------------| -| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | -| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | -| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | -| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | -| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | -| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | -| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | -| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | -| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | -| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | -| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | -| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | -| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | -| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | -| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | -| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | -| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | -| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | -| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | -| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | -| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | -| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | -| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | -| [karpenter\_event\_rules](#output\_karpenter\_event\_rules) | Map of the event rules created and their attributes | -| [karpenter\_iam\_role\_arn](#output\_karpenter\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the controller IAM role | -| [karpenter\_iam\_role\_name](#output\_karpenter\_iam\_role\_name) | The name of the controller IAM role | -| [karpenter\_iam\_role\_unique\_id](#output\_karpenter\_iam\_role\_unique\_id) | Stable and unique string identifying the controller IAM role | -| [karpenter\_instance\_profile\_arn](#output\_karpenter\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | -| [karpenter\_instance\_profile\_id](#output\_karpenter\_instance\_profile\_id) | Instance profile's ID | -| [karpenter\_instance\_profile\_name](#output\_karpenter\_instance\_profile\_name) | Name of the instance profile | -| [karpenter\_instance\_profile\_unique](#output\_karpenter\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | -| [karpenter\_node\_iam\_role\_arn](#output\_karpenter\_node\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | -| [karpenter\_node\_iam\_role\_name](#output\_karpenter\_node\_iam\_role\_name) | The name of the IAM role | -| [karpenter\_node\_iam\_role\_unique\_id](#output\_karpenter\_node\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [karpenter\_queue\_arn](#output\_karpenter\_queue\_arn) | The ARN of the SQS queue | -| [karpenter\_queue\_name](#output\_karpenter\_queue\_name) | The name of the created Amazon SQS queue | -| [karpenter\_queue\_url](#output\_karpenter\_queue\_url) | The URL for the created Amazon SQS queue | -| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | -| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | -| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | -| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | -| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | -| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | +No outputs. diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 49321c977e..9a6223e253 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -35,7 +35,14 @@ provider "kubectl" { } } -data "aws_availability_zones" "available" {} +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + data "aws_ecrpublic_authorization_token" "token" { provider = aws.virginia } diff --git a/examples/karpenter/outputs.tf b/examples/karpenter/outputs.tf index d85a8b034e..e69de29bb2 100644 --- a/examples/karpenter/outputs.tf +++ b/examples/karpenter/outputs.tf @@ -1,297 +0,0 @@ -################################################################################ -# Cluster -################################################################################ - -output "cluster_arn" { - description = "The Amazon Resource Name (ARN) of the cluster" - value = module.eks.cluster_arn -} - -output "cluster_certificate_authority_data" { - description = "Base64 encoded certificate data required to communicate with the cluster" - value = module.eks.cluster_certificate_authority_data -} - -output "cluster_endpoint" { - description = "Endpoint for your Kubernetes API server" - value = module.eks.cluster_endpoint -} - -output "cluster_id" { - description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" - value = module.eks.cluster_id -} - -output "cluster_name" { - description = "The name of the EKS cluster" - value = module.eks.cluster_name -} - -output "cluster_oidc_issuer_url" { - description = "The URL on the EKS cluster for the OpenID Connect identity provider" - value = module.eks.cluster_oidc_issuer_url -} - -output "cluster_dualstack_oidc_issuer_url" { - description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" - value = module.eks.cluster_dualstack_oidc_issuer_url -} - -output "cluster_platform_version" { - description = "Platform version for the cluster" - value = module.eks.cluster_platform_version -} - -output "cluster_status" { - description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" - value = module.eks.cluster_status -} - -output "cluster_primary_security_group_id" { - description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" - value = module.eks.cluster_primary_security_group_id -} - -output "cluster_service_cidr" { - description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" - value = module.eks.cluster_service_cidr -} - -output "cluster_ip_family" { - description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" - value = module.eks.cluster_ip_family -} - -################################################################################ -# Access Entry -################################################################################ - -output "access_entries" { - description = "Map of access entries created and their attributes" - value = module.eks.access_entries -} - -################################################################################ -# Security Group -################################################################################ - -output "cluster_security_group_arn" { - description = "Amazon Resource Name (ARN) of the cluster security group" - value = module.eks.cluster_security_group_arn -} - -output "cluster_security_group_id" { - description = "ID of the cluster security group" - value = module.eks.cluster_security_group_id -} - -################################################################################ -# Node Security Group -################################################################################ - -output "node_security_group_arn" { - description = "Amazon Resource Name (ARN) of the node shared security group" - value = module.eks.node_security_group_arn -} - -output "node_security_group_id" { - description = "ID of the node shared security group" - value = module.eks.node_security_group_id -} - -################################################################################ -# IRSA -################################################################################ - -output "oidc_provider" { - description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" - value = module.eks.oidc_provider -} - -output "oidc_provider_arn" { - description = "The ARN of the OIDC Provider if `enable_irsa = true`" - value = module.eks.oidc_provider_arn -} - -output "cluster_tls_certificate_sha1_fingerprint" { - description = "The SHA1 fingerprint of the public key of the cluster's certificate" - value = module.eks.cluster_tls_certificate_sha1_fingerprint -} - -################################################################################ -# IAM Role -################################################################################ - -output "cluster_iam_role_name" { - description = "IAM role name of the EKS cluster" - value = module.eks.cluster_iam_role_name -} - -output "cluster_iam_role_arn" { - description = "IAM role ARN of the EKS cluster" - value = module.eks.cluster_iam_role_arn -} - -output "cluster_iam_role_unique_id" { - description = "Stable and unique string identifying the IAM role" - value = module.eks.cluster_iam_role_unique_id -} - -################################################################################ -# EKS Addons -################################################################################ - -output "cluster_addons" { - description = "Map of attribute maps for all EKS cluster addons enabled" - value = module.eks.cluster_addons -} - -################################################################################ -# EKS Identity Provider -################################################################################ - -output "cluster_identity_providers" { - description = "Map of attribute maps for all EKS identity providers enabled" - value = module.eks.cluster_identity_providers -} - -################################################################################ -# CloudWatch Log Group -################################################################################ - -output "cloudwatch_log_group_name" { - description = "Name of cloudwatch log group created" - value = module.eks.cloudwatch_log_group_name -} - -output "cloudwatch_log_group_arn" { - description = "Arn of cloudwatch log group created" - value = module.eks.cloudwatch_log_group_arn -} - -################################################################################ -# Fargate Profile -################################################################################ - -output "fargate_profiles" { - description = "Map of attribute maps for all EKS Fargate Profiles created" - value = module.eks.fargate_profiles -} - -################################################################################ -# EKS Managed Node Group -################################################################################ - -output "eks_managed_node_groups" { - description = "Map of attribute maps for all EKS managed node groups created" - value = module.eks.eks_managed_node_groups -} - -output "eks_managed_node_groups_autoscaling_group_names" { - description = "List of the autoscaling group names created by EKS managed node groups" - value = module.eks.eks_managed_node_groups_autoscaling_group_names -} - -################################################################################ -# Self Managed Node Group -################################################################################ - -output "self_managed_node_groups" { - description = "Map of attribute maps for all self managed node groups created" - value = module.eks.self_managed_node_groups -} - -output "self_managed_node_groups_autoscaling_group_names" { - description = "List of the autoscaling group names created by self-managed node groups" - value = module.eks.self_managed_node_groups_autoscaling_group_names -} - -################################################################################ -# Karpenter controller IAM Role -################################################################################ - -output "karpenter_iam_role_name" { - description = "The name of the controller IAM role" - value = module.karpenter.iam_role_name -} - -output "karpenter_iam_role_arn" { - description = "The Amazon Resource Name (ARN) specifying the controller IAM role" - value = module.karpenter.iam_role_arn -} - -output "karpenter_iam_role_unique_id" { - description = "Stable and unique string identifying the controller IAM role" - value = module.karpenter.iam_role_unique_id -} - -################################################################################ -# Node Termination Queue -################################################################################ - -output "karpenter_queue_arn" { - description = "The ARN of the SQS queue" - value = module.karpenter.queue_arn -} - -output "karpenter_queue_name" { - description = "The name of the created Amazon SQS queue" - value = module.karpenter.queue_name -} - -output "karpenter_queue_url" { - description = "The URL for the created Amazon SQS queue" - value = module.karpenter.queue_url -} - -################################################################################ -# Node Termination Event Rules -################################################################################ - -output "karpenter_event_rules" { - description = "Map of the event rules created and their attributes" - value = module.karpenter.event_rules -} - -################################################################################ -# Node IAM Role -################################################################################ - -output "karpenter_node_iam_role_name" { - description = "The name of the IAM role" - value = module.karpenter.node_iam_role_name -} - -output "karpenter_node_iam_role_arn" { - description = "The Amazon Resource Name (ARN) specifying the IAM role" - value = module.karpenter.node_iam_role_arn -} - -output "karpenter_node_iam_role_unique_id" { - description = "Stable and unique string identifying the IAM role" - value = module.karpenter.node_iam_role_unique_id -} - -################################################################################ -# Node IAM Instance Profile -################################################################################ - -output "karpenter_instance_profile_arn" { - description = "ARN assigned by AWS to the instance profile" - value = module.karpenter.instance_profile_arn -} - -output "karpenter_instance_profile_id" { - description = "Instance profile's ID" - value = module.karpenter.instance_profile_id -} - -output "karpenter_instance_profile_name" { - description = "Name of the instance profile" - value = module.karpenter.instance_profile_name -} - -output "karpenter_instance_profile_unique" { - description = "Stable and unique string identifying the IAM instance profile" - value = module.karpenter.instance_profile_unique -} diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 5caab8394a..5d3e660ea4 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/main.tf b/examples/self-managed-node-group/main.tf index ed6982aa58..f8efaeb389 100644 --- a/examples/self-managed-node-group/main.tf +++ b/examples/self-managed-node-group/main.tf @@ -2,7 +2,13 @@ provider "aws" { region = local.region } -data "aws_availability_zones" "available" {} +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} locals { name = "ex-self-mng" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index 0099e6baaf..5b36c201aa 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } } } diff --git a/main.tf b/main.tf index 50f4323c24..c2889b0dd4 100644 --- a/main.tf +++ b/main.tf @@ -18,6 +18,9 @@ locals { create_outposts_local_cluster = length(var.outpost_config) > 0 enable_cluster_encryption_config = length(var.cluster_encryption_config) > 0 && !local.create_outposts_local_cluster + + auto_mode_enabled = try(var.cluster_compute_config.enabled, false) + auto_mode_nodepools_enabled = length(try(var.cluster_compute_config.node_pools, [])) > 0 } ################################################################################ @@ -31,7 +34,7 @@ resource "aws_eks_cluster" "this" { role_arn = local.cluster_role version = var.cluster_version enabled_cluster_log_types = var.cluster_enabled_log_types - bootstrap_self_managed_addons = var.bootstrap_self_managed_addons + bootstrap_self_managed_addons = local.auto_mode_enabled ? coalesce(var.bootstrap_self_managed_addons, false) : var.bootstrap_self_managed_addons access_config { authentication_mode = var.authentication_mode @@ -44,6 +47,16 @@ resource "aws_eks_cluster" "this" { bootstrap_cluster_creator_admin_permissions = false } + dynamic "compute_config" { + for_each = length(var.cluster_compute_config) > 0 ? [var.cluster_compute_config] : [] + + content { + enabled = try(compute_config.value.enabled, null) + node_pools = local.auto_mode_enabled ? try(compute_config.value.node_pools, []) : null + node_role_arn = local.auto_mode_enabled ? try(compute_config.value.node_role_arn, aws_iam_role.eks_auto[0].arn, null) : null + } + } + vpc_config { security_group_ids = compact(distinct(concat(var.cluster_additional_security_group_ids, [local.cluster_security_group_id]))) subnet_ids = coalescelist(var.control_plane_subnet_ids, var.subnet_ids) @@ -57,6 +70,14 @@ resource "aws_eks_cluster" "this" { for_each = local.create_outposts_local_cluster ? [] : [1] content { + dynamic "elastic_load_balancing" { + for_each = local.auto_mode_enabled ? [1] : [] + + content { + enabled = local.auto_mode_enabled + } + } + ip_family = var.cluster_ip_family service_ipv4_cidr = var.cluster_service_ipv4_cidr service_ipv6_cidr = var.cluster_service_ipv6_cidr @@ -84,6 +105,39 @@ resource "aws_eks_cluster" "this" { } } + dynamic "remote_network_config" { + # Not valid on Outposts + for_each = length(var.cluster_remote_network_config) > 0 && !local.create_outposts_local_cluster ? [var.cluster_remote_network_config] : [] + + content { + dynamic "remote_node_networks" { + for_each = [remote_network_config.value.remote_node_networks] + + content { + cidrs = remote_node_networks.value.cidrs + } + } + + dynamic "remote_pod_networks" { + for_each = try([remote_network_config.value.remote_pod_networks], []) + + content { + cidrs = remote_pod_networks.value.cidrs + } + } + } + } + + dynamic "storage_config" { + for_each = local.auto_mode_enabled ? [1] : [] + + content { + block_storage { + enabled = local.auto_mode_enabled + } + } + } + dynamic "upgrade_policy" { for_each = length(var.cluster_upgrade_policy) > 0 ? [var.cluster_upgrade_policy] : [] @@ -199,7 +253,7 @@ locals { association_policy_arn = pol_val.policy_arn association_access_scope_type = pol_val.access_scope.type association_access_scope_namespaces = lookup(pol_val.access_scope, "namespaces", []) - } : k => v if !contains(["EC2_LINUX", "EC2_WINDOWS", "FARGATE_LINUX"], lookup(entry_val, "type", "STANDARD")) }, + } : k => v if !contains(["EC2", "EC2_LINUX", "EC2_WINDOWS", "FARGATE_LINUX", "HYBRID_LINUX"], lookup(entry_val, "type", "STANDARD")) }, ) ] ]) @@ -380,14 +434,41 @@ locals { iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" cluster_encryption_policy_name = coalesce(var.cluster_encryption_policy_name, "${local.iam_role_name}-ClusterEncryption") + + # Standard EKS cluster + eks_standard_iam_role_policies = { for k, v in { + AmazonEKSClusterPolicy = "${local.iam_role_policy_prefix}/AmazonEKSClusterPolicy", + } : k => v if !local.create_outposts_local_cluster && !local.auto_mode_enabled } + + # EKS cluster with EKS auto mode enabled + eks_auto_mode_iam_role_policies = { for k, v in { + AmazonEKSClusterPolicy = "${local.iam_role_policy_prefix}/AmazonEKSClusterPolicy" + AmazonEKSComputePolicy = "${local.iam_role_policy_prefix}/AmazonEKSComputePolicy" + AmazonEKSBlockStoragePolicy = "${local.iam_role_policy_prefix}/AmazonEKSBlockStoragePolicy" + AmazonEKSLoadBalancingPolicy = "${local.iam_role_policy_prefix}/AmazonEKSLoadBalancingPolicy" + AmazonEKSNetworkingPolicy = "${local.iam_role_policy_prefix}/AmazonEKSNetworkingPolicy" + } : k => v if !local.create_outposts_local_cluster && local.auto_mode_enabled } + + # EKS local cluster on Outposts + eks_outpost_iam_role_policies = { for k, v in { + AmazonEKSClusterPolicy = "${local.iam_role_policy_prefix}/AmazonEKSLocalOutpostClusterPolicy" + } : k => v if local.create_outposts_local_cluster && !local.auto_mode_enabled } + + # Security groups for pods + eks_sgpp_iam_role_policies = { for k, v in { + AmazonEKSVPCResourceController = "${local.iam_role_policy_prefix}/AmazonEKSVPCResourceController" + } : k => v if var.enable_security_groups_for_pods && !local.create_outposts_local_cluster && !local.auto_mode_enabled } } data "aws_iam_policy_document" "assume_role_policy" { count = local.create && var.create_iam_role ? 1 : 0 statement { - sid = "EKSClusterAssumeRole" - actions = ["sts:AssumeRole"] + sid = "EKSClusterAssumeRole" + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] principals { type = "Service" @@ -398,10 +479,8 @@ data "aws_iam_policy_document" "assume_role_policy" { for_each = local.create_outposts_local_cluster ? [1] : [] content { - type = "Service" - identifiers = [ - "ec2.amazonaws.com", - ] + type = "Service" + identifiers = ["ec2.amazonaws.com"] } } } @@ -424,10 +503,12 @@ resource "aws_iam_role" "this" { # Policies attached ref https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html resource "aws_iam_role_policy_attachment" "this" { - for_each = { for k, v in { - AmazonEKSClusterPolicy = local.create_outposts_local_cluster ? "${local.iam_role_policy_prefix}/AmazonEKSLocalOutpostClusterPolicy" : "${local.iam_role_policy_prefix}/AmazonEKSClusterPolicy", - AmazonEKSVPCResourceController = "${local.iam_role_policy_prefix}/AmazonEKSVPCResourceController", - } : k => v if local.create_iam_role } + for_each = { for k, v in merge( + local.eks_standard_iam_role_policies, + local.eks_auto_mode_iam_role_policies, + local.eks_outpost_iam_role_policies, + local.eks_sgpp_iam_role_policies, + ) : k => v if local.create_iam_role } policy_arn = each.value role = aws_iam_role.this[0].name @@ -602,3 +683,218 @@ resource "aws_eks_identity_provider_config" "this" { tags = merge(var.tags, try(each.value.tags, {})) } + +################################################################################ +# EKS Auto Node IAM Role +################################################################################ + +locals { + create_node_iam_role = local.create && var.create_node_iam_role && local.auto_mode_nodepools_enabled + node_iam_role_name = coalesce(var.node_iam_role_name, "${var.cluster_name}-eks-auto") + + create_node_iam_role_custom_policy = local.create_node_iam_role && (var.enable_node_custom_tags_permissions || length(var.node_iam_role_policy_statements) > 0) +} + +data "aws_iam_policy_document" "node_assume_role_policy" { + count = local.create_node_iam_role ? 1 : 0 + + statement { + sid = "EKSAutoNodeAssumeRole" + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "eks_auto" { + count = local.create_node_iam_role ? 1 : 0 + + name = var.node_iam_role_use_name_prefix ? null : local.node_iam_role_name + name_prefix = var.node_iam_role_use_name_prefix ? "${local.node_iam_role_name}-" : null + path = var.node_iam_role_path + description = var.node_iam_role_description + + assume_role_policy = data.aws_iam_policy_document.node_assume_role_policy[0].json + permissions_boundary = var.node_iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.node_iam_role_tags) +} + +# Policies attached ref https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html +resource "aws_iam_role_policy_attachment" "eks_auto" { + for_each = { for k, v in { + AmazonEKSWorkerNodeMinimalPolicy = "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodeMinimalPolicy", + AmazonEC2ContainerRegistryPullOnly = "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryPullOnly", + } : k => v if local.create_node_iam_role } + + policy_arn = each.value + role = aws_iam_role.eks_auto[0].name +} + +resource "aws_iam_role_policy_attachment" "eks_auto_additional" { + for_each = { for k, v in var.node_iam_role_additional_policies : k => v if local.create_node_iam_role } + + policy_arn = each.value + role = aws_iam_role.eks_auto[0].name +} + +resource "aws_iam_role_policy_attachment" "eks_auto_custom" { + count = local.create_node_iam_role_custom_policy ? 1 : 0 + + policy_arn = aws_iam_policy.eks_auto_custom[0].arn + role = aws_iam_role.eks_auto[0].name +} + +data "aws_iam_policy_document" "eks_auto_custom" { + count = local.create_node_iam_role_custom_policy ? 1 : 0 + + dynamic "statement" { + for_each = var.enable_node_custom_tags_permissions ? [1] : [] + + content { + sid = "Compute" + actions = [ + "ec2:CreateFleet", + "ec2:RunInstances", + "ec2:CreateLaunchTemplate", + ] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/eks:kubernetes-node-class-name" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/eks:kubernetes-node-pool-name" + values = ["*"] + } + } + } + + dynamic "statement" { + for_each = var.enable_node_custom_tags_permissions ? [1] : [] + + content { + sid = "Storage" + actions = [ + "ec2:CreateVolume", + "ec2:CreateSnapshot", + ] + resources = [ + "arn:${local.partition}:ec2:*:*:volume/*", + "arn:${local.partition}:ec2:*:*:snapshot/*", + ] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } + + dynamic "statement" { + for_each = var.enable_node_custom_tags_permissions ? [1] : [] + + content { + sid = "Networking" + actions = ["ec2:CreateNetworkInterface"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:kubernetes-cni-node-name" + values = ["*"] + } + } + } + + dynamic "statement" { + for_each = var.enable_node_custom_tags_permissions ? [1] : [] + + content { + sid = "LoadBalancer" + actions = [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateRule", + "ec2:CreateSecurityGroup", + ] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } + + dynamic "statement" { + for_each = var.enable_node_custom_tags_permissions ? [1] : [] + + content { + sid = "ShieldProtection" + actions = ["shield:CreateProtection"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } + + dynamic "statement" { + for_each = var.enable_node_custom_tags_permissions ? [1] : [] + + content { + sid = "ShieldTagResource" + actions = ["shield:TagResource"] + resources = ["arn:${local.partition}:shield::*:protection/*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } +} + +resource "aws_iam_policy" "eks_auto_custom" { + count = local.create_node_iam_role_custom_policy ? 1 : 0 + + name = var.node_iam_role_use_name_prefix ? null : local.node_iam_role_name + name_prefix = var.node_iam_role_use_name_prefix ? "${local.node_iam_role_name}-" : null + path = var.node_iam_role_path + description = var.node_iam_role_description + + policy = data.aws_iam_policy_document.eks_auto_custom[0].json + + tags = merge(var.tags, var.node_iam_role_tags) +} diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 23df973444..0936b8f790 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.75 | +| [aws](#requirement\_aws) | >= 5.79 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.75 | +| [aws](#provider\_aws) | >= 5.79 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 0099e6baaf..5b36c201aa 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index a7b12553ff..0d47f4697a 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.75 | +| [aws](#requirement\_aws) | >= 5.79 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.75 | +| [aws](#provider\_aws) | >= 5.79 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 0099e6baaf..5b36c201aa 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md new file mode 100644 index 0000000000..bd93c4ba1d --- /dev/null +++ b/modules/hybrid-node-role/README.md @@ -0,0 +1,159 @@ +# EKS Hybrid Node Role Module + +Terraform module which creates IAM role and policy resources for Amazon EKS Hybrid Node(s). + +## Usage + +EKS Hybrid nodes use the AWS IAM Authenticator and temporary IAM credentials provisioned by AWS SSM or AWS IAM Roles Anywhere to authenticate with the EKS cluster. This module supports both SSM and IAM Roles Anywhere based IAM permissions. + +### SSM + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + + ... + access_entries = { + hybrid-node-role = { + principal_arn = module.eks_hybrid_node_role.arn + type = "HYBRID_LINUX" + } + } +} + +module "eks_hybrid_node_role" { + source = "terraform-aws-modules/eks/aws//modules/hybrid-node-role" + + name = "hybrid" + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### IAM Roles Anywhere + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + + ... + access_entries = { + hybrid-node-role = { + principal_arn = module.eks_hybrid_node_role.arn + type = "HYBRID_LINUX" + } + } +} + +module "eks_hybrid_node_role" { + source = "terraform-aws-modules/eks/aws//modules/hybrid-node-role" + + name = "hybrid-ira" + + enable_ira = true + + ira_trust_anchor_source_type = "CERTIFICATE_BUNDLE" + ira_trust_anchor_x509_certificate_data = <<-EOT + MIIFMzCCAxugAwIBAgIRAMnVXU7ncv/+Cl16eJbZ9hswDQYJKoZIhvcNAQELBQAw + ... + MGx/BMRkrNUVcg3xA0lhECo/olodCkmZo5/mjybbjFQwJzDSKFoW + EOT + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [aws](#requirement\_aws) | >= 5.79 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.79 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_policy.intermediate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.intermediate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.intermediate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_rolesanywhere_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rolesanywhere_profile) | resource | +| [aws_rolesanywhere_trust_anchor.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rolesanywhere_trust_anchor) | resource | +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.intermediate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.intermediate_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_arns](#input\_cluster\_arns) | List of EKS cluster ARNs to allow the node to describe | `list(string)` |
[
"*"
]
| no | +| [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | +| [description](#input\_description) | IAM role description | `string` | `"EKS Hybrid Node IAM role"` | no | +| [enable\_ira](#input\_enable\_ira) | Enables IAM Roles Anywhere based IAM permissions on the node | `bool` | `false` | no | +| [enable\_pod\_identity](#input\_enable\_pod\_identity) | Enables EKS Pod Identity based IAM permissions on the node | `bool` | `true` | no | +| [intermediate\_policy\_name](#input\_intermediate\_policy\_name) | Name of the IAM policy | `string` | `null` | no | +| [intermediate\_policy\_statements](#input\_intermediate\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [intermediate\_policy\_use\_name\_prefix](#input\_intermediate\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`intermediate_policy_name`) is used as a prefix | `bool` | `true` | no | +| [intermediate\_role\_description](#input\_intermediate\_role\_description) | IAM role description | `string` | `"EKS Hybrid Node IAM Roles Anywhere intermediate IAM role"` | no | +| [intermediate\_role\_name](#input\_intermediate\_role\_name) | Name of the IAM role | `string` | `null` | no | +| [intermediate\_role\_path](#input\_intermediate\_role\_path) | Path of the IAM role | `string` | `"/"` | no | +| [intermediate\_role\_policies](#input\_intermediate\_role\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | +| [intermediate\_role\_use\_name\_prefix](#input\_intermediate\_role\_use\_name\_prefix) | Determines whether the name of the IAM role (`intermediate_role_name`) is used as a prefix | `bool` | `true` | no | +| [ira\_profile\_duration\_seconds](#input\_ira\_profile\_duration\_seconds) | The number of seconds the vended session credentials are valid for. Defaults to `3600` | `number` | `null` | no | +| [ira\_profile\_managed\_policy\_arns](#input\_ira\_profile\_managed\_policy\_arns) | A list of managed policy ARNs that apply to the vended session credentials | `list(string)` | `[]` | no | +| [ira\_profile\_name](#input\_ira\_profile\_name) | Name of the Roles Anywhere profile | `string` | `null` | no | +| [ira\_profile\_require\_instance\_properties](#input\_ira\_profile\_require\_instance\_properties) | Specifies whether instance properties are required in [CreateSession](https://docs.aws.amazon.com/rolesanywhere/latest/APIReference/API_CreateSession.html) requests with this profile | `bool` | `null` | no | +| [ira\_profile\_session\_policy](#input\_ira\_profile\_session\_policy) | A session policy that applies to the trust boundary of the vended session credentials | `string` | `null` | no | +| [ira\_trust\_anchor\_acm\_pca\_arn](#input\_ira\_trust\_anchor\_acm\_pca\_arn) | The ARN of the ACM PCA that issued the trust anchor certificate | `string` | `null` | no | +| [ira\_trust\_anchor\_name](#input\_ira\_trust\_anchor\_name) | Name of the Roles Anywhere trust anchor | `string` | `null` | no | +| [ira\_trust\_anchor\_notification\_settings](#input\_ira\_trust\_anchor\_notification\_settings) | Notification settings for the trust anchor | `any` | `[]` | no | +| [ira\_trust\_anchor\_source\_type](#input\_ira\_trust\_anchor\_source\_type) | The source type of the trust anchor | `string` | `null` | no | +| [ira\_trust\_anchor\_x509\_certificate\_data](#input\_ira\_trust\_anchor\_x509\_certificate\_data) | The X.509 certificate data of the trust anchor | `string` | `null` | no | +| [max\_session\_duration](#input\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | +| [name](#input\_name) | Name of the IAM role | `string` | `"EKSHybridNode"` | no | +| [path](#input\_path) | Path of the IAM role | `string` | `"/"` | no | +| [permissions\_boundary\_arn](#input\_permissions\_boundary\_arn) | Permissions boundary ARN to use for the IAM role | `string` | `null` | no | +| [policies](#input\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | +| [policy\_description](#input\_policy\_description) | IAM policy description | `string` | `"EKS Hybrid Node IAM role policy"` | no | +| [policy\_name](#input\_policy\_name) | Name of the IAM policy | `string` | `"EKSHybridNode"` | no | +| [policy\_path](#input\_policy\_path) | Path of the IAM policy | `string` | `"/"` | no | +| [policy\_statements](#input\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [policy\_use\_name\_prefix](#input\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`policy_name`) is used as a prefix | `bool` | `true` | no | +| [tags](#input\_tags) | A map of additional tags to add the the IAM role | `map(any)` | `{}` | no | +| [trust\_anchor\_arns](#input\_trust\_anchor\_arns) | List of IAM Roles Anywhere trust anchor ARNs. Required if `enable_ira` is set to `true` | `list(string)` | `[]` | no | +| [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether the name of the IAM role (`name`) is used as a prefix | `bool` | `true` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [arn](#output\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | +| [intermediate\_role\_arn](#output\_intermediate\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | +| [intermediate\_role\_name](#output\_intermediate\_role\_name) | The name of the node IAM role | +| [intermediate\_role\_unique\_id](#output\_intermediate\_role\_unique\_id) | Stable and unique string identifying the node IAM role | +| [name](#output\_name) | The name of the node IAM role | +| [unique\_id](#output\_unique\_id) | Stable and unique string identifying the node IAM role | + diff --git a/modules/hybrid-node-role/main.tf b/modules/hybrid-node-role/main.tf new file mode 100644 index 0000000000..5a2473f29b --- /dev/null +++ b/modules/hybrid-node-role/main.tf @@ -0,0 +1,369 @@ +data "aws_partition" "current" { + count = var.create ? 1 : 0 +} + +locals { + partition = try(data.aws_partition.current[0].partition, "aws") +} + +################################################################################ +# Node IAM Role +################################################################################ + +data "aws_iam_policy_document" "assume_role" { + count = var.create ? 1 : 0 + + # SSM + dynamic "statement" { + for_each = var.enable_ira ? [] : [1] + + content { + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "Service" + identifiers = ["ssm.amazonaws.com"] + } + } + } + + # IAM Roles Anywhere + dynamic "statement" { + for_each = var.enable_ira ? [1] : [] + + content { + actions = [ + "sts:TagSession", + "sts:SetSourceIdentity", + ] + + principals { + type = "AWS" + identifiers = [aws_iam_role.intermediate[0].arn] + } + } + } + + dynamic "statement" { + for_each = var.enable_ira ? [1] : [] + + content { + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "AWS" + identifiers = [aws_iam_role.intermediate[0].arn] + } + + condition { + test = "StringEquals" + variable = "sts:RoleSessionName" + values = ["$${aws:PrincipalTag/x509Subject/CN}"] + } + } + } +} + +resource "aws_iam_role" "this" { + count = var.create ? 1 : 0 + + name = var.use_name_prefix ? null : var.name + name_prefix = var.use_name_prefix ? "${var.name}-" : null + path = var.path + description = var.description + + assume_role_policy = data.aws_iam_policy_document.assume_role[0].json + max_session_duration = var.max_session_duration + permissions_boundary = var.permissions_boundary_arn + force_detach_policies = true + + tags = var.tags +} + +################################################################################ +# Node IAM Role Policy +################################################################################ + +data "aws_iam_policy_document" "this" { + count = var.create ? 1 : 0 + + statement { + actions = [ + "ssm:DeregisterManagedInstance", + "ssm:DescribeInstanceInformation", + ] + + resources = ["*"] + } + + statement { + actions = ["eks:DescribeCluster"] + resources = var.cluster_arns + } + + dynamic "statement" { + for_each = var.enable_pod_identity ? [1] : [] + + content { + actions = ["eks-auth:AssumeRoleForPodIdentity"] + resources = ["*"] + } + } + + dynamic "statement" { + for_each = var.policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +resource "aws_iam_policy" "this" { + count = var.create ? 1 : 0 + + name = var.policy_use_name_prefix ? null : var.policy_name + name_prefix = var.policy_use_name_prefix ? "${var.policy_name}-" : null + path = var.policy_path + description = var.policy_description + policy = data.aws_iam_policy_document.this[0].json + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "this" { + for_each = { for k, v in merge( + { + node = try(aws_iam_policy.this[0].arn, null) + AmazonSSMManagedInstanceCore = "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + AmazonEC2ContainerRegistryPullOnly = "arn:${local.partition}:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly" + }, + var.policies + ) : k => v if var.create } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +################################################################################ +# Roles Anywhere Profile +################################################################################ + +locals { + enable_ira = var.create && var.enable_ira +} + +resource "aws_rolesanywhere_profile" "this" { + count = local.enable_ira ? 1 : 0 + + duration_seconds = var.ira_profile_duration_seconds + managed_policy_arns = var.ira_profile_managed_policy_arns + name = try(coalesce(var.ira_profile_name, var.name), null) + require_instance_properties = var.ira_profile_require_instance_properties + role_arns = [aws_iam_role.intermediate[0].arn] + session_policy = var.ira_profile_session_policy + + tags = var.tags +} + +################################################################################ +# Roles Anywhere Trust Anchor +################################################################################ + +resource "aws_rolesanywhere_trust_anchor" "this" { + count = local.enable_ira ? 1 : 0 + + name = try(coalesce(var.ira_trust_anchor_name, var.name), null) + + dynamic "notification_settings" { + for_each = var.ira_trust_anchor_notification_settings + + content { + channel = try(notification_settings.value.channel, null) + enabled = try(notification_settings.value.enabled, null) + event = try(notification_settings.value.event, null) + threshold = try(notification_settings.value.threshold, null) + } + } + + source { + source_data { + acm_pca_arn = var.ira_trust_anchor_acm_pca_arn + x509_certificate_data = var.ira_trust_anchor_x509_certificate_data + } + source_type = var.ira_trust_anchor_source_type + } + + tags = var.tags +} + +################################################################################ +# Intermediate IAM Role +################################################################################ + +data "aws_iam_policy_document" "intermediate_assume_role" { + count = local.enable_ira ? 1 : 0 + + statement { + actions = [ + "sts:AssumeRole", + "sts:TagSession", + "sts:SetSourceIdentity", + ] + + principals { + type = "Service" + identifiers = ["rolesanywhere.amazonaws.com"] + } + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = concat(var.trust_anchor_arns, aws_rolesanywhere_trust_anchor.this[*].arn) + } + } +} + +locals { + intermediate_role_use_name_prefix = coalesce(var.intermediate_role_use_name_prefix, var.use_name_prefix) + intermediate_role_name = coalesce(var.intermediate_role_name, "${var.name}-inter") +} + +resource "aws_iam_role" "intermediate" { + count = local.enable_ira ? 1 : 0 + + name = local.intermediate_role_use_name_prefix ? null : local.intermediate_role_name + name_prefix = local.intermediate_role_use_name_prefix ? "${local.intermediate_role_name}-" : null + path = coalesce(var.intermediate_role_path, var.path) + description = var.intermediate_role_description + + assume_role_policy = data.aws_iam_policy_document.intermediate_assume_role[0].json + max_session_duration = var.max_session_duration + permissions_boundary = var.permissions_boundary_arn + force_detach_policies = true + + tags = var.tags +} + +################################################################################ +# Intermediate IAM Role Policy +################################################################################ + +data "aws_iam_policy_document" "intermediate" { + count = local.enable_ira ? 1 : 0 + + statement { + actions = ["eks:DescribeCluster"] + resources = var.cluster_arns + } + + dynamic "statement" { + for_each = var.intermediate_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +locals { + intermediate_policy_use_name_prefix = coalesce(var.intermediate_policy_use_name_prefix, var.policy_use_name_prefix) + intermediate_policy_name = coalesce(var.intermediate_policy_name, var.policy_name) +} + +resource "aws_iam_policy" "intermediate" { + count = local.enable_ira ? 1 : 0 + + name = local.intermediate_policy_use_name_prefix ? null : local.intermediate_policy_name + name_prefix = local.intermediate_policy_use_name_prefix ? "${local.intermediate_policy_name}-" : null + path = var.policy_path + description = var.policy_description + policy = data.aws_iam_policy_document.intermediate[0].json + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "intermediate" { + for_each = { for k, v in merge( + { + intermediate = try(aws_iam_policy.intermediate[0].arn, null) + AmazonEC2ContainerRegistryPullOnly = "arn:${local.partition}:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly" + }, + var.intermediate_role_policies + ) : k => v if local.enable_ira } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} diff --git a/modules/hybrid-node-role/outputs.tf b/modules/hybrid-node-role/outputs.tf new file mode 100644 index 0000000000..dc4e26e519 --- /dev/null +++ b/modules/hybrid-node-role/outputs.tf @@ -0,0 +1,37 @@ +################################################################################ +# Node IAM Role +################################################################################ + +output "name" { + description = "The name of the node IAM role" + value = try(aws_iam_role.this[0].name, null) +} + +output "arn" { + description = "The Amazon Resource Name (ARN) specifying the node IAM role" + value = try(aws_iam_role.this[0].arn, null) +} + +output "unique_id" { + description = "Stable and unique string identifying the node IAM role" + value = try(aws_iam_role.this[0].unique_id, null) +} + +################################################################################ +# Intermedaite IAM Role +################################################################################ + +output "intermediate_role_name" { + description = "The name of the node IAM role" + value = try(aws_iam_role.intermediate[0].name, null) +} + +output "intermediate_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the node IAM role" + value = try(aws_iam_role.intermediate[0].arn, null) +} + +output "intermediate_role_unique_id" { + description = "Stable and unique string identifying the node IAM role" + value = try(aws_iam_role.intermediate[0].unique_id, null) +} diff --git a/modules/hybrid-node-role/variables.tf b/modules/hybrid-node-role/variables.tf new file mode 100644 index 0000000000..6e8d80465e --- /dev/null +++ b/modules/hybrid-node-role/variables.tf @@ -0,0 +1,239 @@ +variable "create" { + description = "Controls if resources should be created (affects nearly all resources)" + type = bool + default = true +} + +################################################################################ +# Node IAM Role +################################################################################ + +variable "name" { + description = "Name of the IAM role" + type = string + default = "EKSHybridNode" +} + +variable "use_name_prefix" { + description = "Determines whether the name of the IAM role (`name`) is used as a prefix" + type = bool + default = true +} + +variable "path" { + description = "Path of the IAM role" + type = string + default = "/" +} + +variable "description" { + description = "IAM role description" + type = string + default = "EKS Hybrid Node IAM role" +} + +variable "max_session_duration" { + description = "Maximum API session duration in seconds between 3600 and 43200" + type = number + default = null +} + +variable "permissions_boundary_arn" { + description = "Permissions boundary ARN to use for the IAM role" + type = string + default = null +} + +variable "tags" { + description = "A map of additional tags to add the the IAM role" + type = map(any) + default = {} +} + +variable "enable_ira" { + description = "Enables IAM Roles Anywhere based IAM permissions on the node" + type = bool + default = false +} + +variable "trust_anchor_arns" { + description = "List of IAM Roles Anywhere trust anchor ARNs. Required if `enable_ira` is set to `true`" + type = list(string) + default = [] +} + +################################################################################ +# Node IAM Role Policy +################################################################################ + +variable "policy_name" { + description = "Name of the IAM policy" + type = string + default = "EKSHybridNode" +} + +variable "policy_use_name_prefix" { + description = "Determines whether the name of the IAM policy (`policy_name`) is used as a prefix" + type = bool + default = true +} + +variable "policy_path" { + description = "Path of the IAM policy" + type = string + default = "/" +} + +variable "policy_description" { + description = "IAM policy description" + type = string + default = "EKS Hybrid Node IAM role policy" +} + +variable "policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + +variable "policies" { + description = "Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format" + type = map(string) + default = {} +} + +variable "cluster_arns" { + description = "List of EKS cluster ARNs to allow the node to describe" + type = list(string) + default = ["*"] +} + +variable "enable_pod_identity" { + description = "Enables EKS Pod Identity based IAM permissions on the node" + type = bool + default = true +} + +################################################################################ +# IAM Roles Anywhere Profile +################################################################################ + +variable "ira_profile_name" { + description = "Name of the Roles Anywhere profile" + type = string + default = null +} + +variable "ira_profile_duration_seconds" { + description = "The number of seconds the vended session credentials are valid for. Defaults to `3600`" + type = number + default = null +} + +variable "ira_profile_managed_policy_arns" { + description = "A list of managed policy ARNs that apply to the vended session credentials" + type = list(string) + default = [] +} + +variable "ira_profile_require_instance_properties" { + description = "Specifies whether instance properties are required in [CreateSession](https://docs.aws.amazon.com/rolesanywhere/latest/APIReference/API_CreateSession.html) requests with this profile" + type = bool + default = null +} + +variable "ira_profile_session_policy" { + description = "A session policy that applies to the trust boundary of the vended session credentials" + type = string + default = null +} + +################################################################################ +# Roles Anywhere Trust Anchor +################################################################################ + +variable "ira_trust_anchor_name" { + description = "Name of the Roles Anywhere trust anchor" + type = string + default = null +} + +variable "ira_trust_anchor_notification_settings" { + description = "Notification settings for the trust anchor" + type = any + default = [] +} + +variable "ira_trust_anchor_acm_pca_arn" { + description = "The ARN of the ACM PCA that issued the trust anchor certificate" + type = string + default = null +} + +variable "ira_trust_anchor_x509_certificate_data" { + description = "The X.509 certificate data of the trust anchor" + type = string + default = null +} + +variable "ira_trust_anchor_source_type" { + description = "The source type of the trust anchor" + type = string + default = null +} + +################################################################################ +# Intermediate IAM Role +################################################################################ + +variable "intermediate_role_name" { + description = "Name of the IAM role" + type = string + default = null +} + +variable "intermediate_role_use_name_prefix" { + description = "Determines whether the name of the IAM role (`intermediate_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "intermediate_role_path" { + description = "Path of the IAM role" + type = string + default = "/" +} + +variable "intermediate_role_description" { + description = "IAM role description" + type = string + default = "EKS Hybrid Node IAM Roles Anywhere intermediate IAM role" +} + +################################################################################ +# Intermediate IAM Role Policy +################################################################################ + +variable "intermediate_policy_name" { + description = "Name of the IAM policy" + type = string + default = null +} + +variable "intermediate_policy_use_name_prefix" { + description = "Determines whether the name of the IAM policy (`intermediate_policy_name`) is used as a prefix" + type = bool + default = true +} + +variable "intermediate_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + +variable "intermediate_role_policies" { + description = "Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format" + type = map(string) + default = {} +} diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf new file mode 100644 index 0000000000..5b36c201aa --- /dev/null +++ b/modules/hybrid-node-role/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.2" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.79" + } + } +} diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index ef2be2099c..891d1635df 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.75 | +| [aws](#requirement\_aws) | >= 5.79 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.75 | +| [aws](#provider\_aws) | >= 5.79 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 0099e6baaf..5b36c201aa 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 7c76477049..654e223b1e 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.75 | +| [aws](#requirement\_aws) | >= 5.79 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.75 | +| [aws](#provider\_aws) | >= 5.79 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 0099e6baaf..5b36c201aa 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } } } diff --git a/outputs.tf b/outputs.tf index 0bd2d10f4e..688076f604 100644 --- a/outputs.tf +++ b/outputs.tf @@ -176,12 +176,12 @@ output "cluster_tls_certificate_sha1_fingerprint" { ################################################################################ output "cluster_iam_role_name" { - description = "IAM role name of the EKS cluster" + description = "Cluster IAM role name" value = try(aws_iam_role.this[0].name, null) } output "cluster_iam_role_arn" { - description = "IAM role ARN of the EKS cluster" + description = "Cluster IAM role ARN" value = try(aws_iam_role.this[0].arn, null) } @@ -190,6 +190,25 @@ output "cluster_iam_role_unique_id" { value = try(aws_iam_role.this[0].unique_id, null) } +################################################################################ +# EKS Auto Node IAM Role +################################################################################ + +output "node_iam_role_name" { + description = "EKS Auto node IAM role name" + value = try(aws_iam_role.eks_auto[0].name, null) +} + +output "node_iam_role_arn" { + description = "EKS Auto node IAM role ARN" + value = try(aws_iam_role.eks_auto[0].arn, null) +} + +output "node_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.eks_auto[0].unique_id, null) +} + ################################################################################ # EKS Addons ################################################################################ diff --git a/tests/fargate-profile/README.md b/tests/eks-fargate-profile/README.md similarity index 91% rename from tests/fargate-profile/README.md rename to tests/eks-fargate-profile/README.md index a50029c722..9035c4efbf 100644 --- a/tests/fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -1,4 +1,4 @@ -# Fargate Profile +# EKS Fargate Profile ## Usage @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.75 | +| [aws](#requirement\_aws) | >= 5.79 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.75 | +| [aws](#provider\_aws) | >= 5.79 | ## Modules @@ -58,8 +58,8 @@ No inputs. | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | | [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | -| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | -| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | Cluster IAM role ARN | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | Cluster IAM role name | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | @@ -79,6 +79,9 @@ No inputs. | [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | | [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | | [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | +| [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | EKS Auto node IAM role ARN | +| [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | EKS Auto node IAM role name | +| [node\_iam\_role\_unique\_id](#output\_node\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/tests/fargate-profile/main.tf b/tests/eks-fargate-profile/main.tf similarity index 96% rename from tests/fargate-profile/main.tf rename to tests/eks-fargate-profile/main.tf index fc9afc76ed..37fc44b3f9 100644 --- a/tests/fargate-profile/main.tf +++ b/tests/eks-fargate-profile/main.tf @@ -2,7 +2,13 @@ provider "aws" { region = local.region } -data "aws_availability_zones" "available" {} +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} locals { name = "ex-${basename(path.cwd)}" diff --git a/tests/eks-fargate-profile/outputs.tf b/tests/eks-fargate-profile/outputs.tf new file mode 100644 index 0000000000..9ed8c27220 --- /dev/null +++ b/tests/eks-fargate-profile/outputs.tf @@ -0,0 +1,245 @@ +################################################################################ +# Cluster +################################################################################ + +output "cluster_arn" { + description = "The Amazon Resource Name (ARN) of the cluster" + value = module.eks.cluster_arn +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded certificate data required to communicate with the cluster" + value = module.eks.cluster_certificate_authority_data +} + +output "cluster_endpoint" { + description = "Endpoint for your Kubernetes API server" + value = module.eks.cluster_endpoint +} + +output "cluster_id" { + description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" + value = module.eks.cluster_id +} + +output "cluster_name" { + description = "The name of the EKS cluster" + value = module.eks.cluster_name +} + +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_oidc_issuer_url +} + +output "cluster_dualstack_oidc_issuer_url" { + description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_dualstack_oidc_issuer_url +} + +output "cluster_platform_version" { + description = "Platform version for the cluster" + value = module.eks.cluster_platform_version +} + +output "cluster_status" { + description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" + value = module.eks.cluster_status +} + +output "cluster_primary_security_group_id" { + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = module.eks.cluster_primary_security_group_id +} + +output "cluster_service_cidr" { + description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" + value = module.eks.cluster_service_cidr +} + +output "cluster_ip_family" { + description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" + value = module.eks.cluster_ip_family +} + +################################################################################ +# Access Entry +################################################################################ + +output "access_entries" { + description = "Map of access entries created and their attributes" + value = module.eks.access_entries +} + +################################################################################ +# KMS Key +################################################################################ + +output "kms_key_arn" { + description = "The Amazon Resource Name (ARN) of the key" + value = module.eks.kms_key_arn +} + +output "kms_key_id" { + description = "The globally unique identifier for the key" + value = module.eks.kms_key_id +} + +output "kms_key_policy" { + description = "The IAM resource policy set on the key" + value = module.eks.kms_key_policy +} + +################################################################################ +# Security Group +################################################################################ + +output "cluster_security_group_arn" { + description = "Amazon Resource Name (ARN) of the cluster security group" + value = module.eks.cluster_security_group_arn +} + +output "cluster_security_group_id" { + description = "ID of the cluster security group" + value = module.eks.cluster_security_group_id +} + +################################################################################ +# Node Security Group +################################################################################ + +output "node_security_group_arn" { + description = "Amazon Resource Name (ARN) of the node shared security group" + value = module.eks.node_security_group_arn +} + +output "node_security_group_id" { + description = "ID of the node shared security group" + value = module.eks.node_security_group_id +} + +################################################################################ +# IRSA +################################################################################ + +output "oidc_provider" { + description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" + value = module.eks.oidc_provider +} + +output "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + value = module.eks.oidc_provider_arn +} + +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = module.eks.cluster_tls_certificate_sha1_fingerprint +} + +################################################################################ +# IAM Role +################################################################################ + +output "cluster_iam_role_name" { + description = "Cluster IAM role name" + value = module.eks.cluster_iam_role_name +} + +output "cluster_iam_role_arn" { + description = "Cluster IAM role ARN" + value = module.eks.cluster_iam_role_arn +} + +output "cluster_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.cluster_iam_role_unique_id +} + +################################################################################ +# EKS Auto Node IAM Role +################################################################################ + +output "node_iam_role_name" { + description = "EKS Auto node IAM role name" + value = module.eks.node_iam_role_name +} + +output "node_iam_role_arn" { + description = "EKS Auto node IAM role ARN" + value = module.eks.node_iam_role_arn +} + +output "node_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.node_iam_role_unique_id +} + +################################################################################ +# EKS Addons +################################################################################ + +output "cluster_addons" { + description = "Map of attribute maps for all EKS cluster addons enabled" + value = module.eks.cluster_addons +} + +################################################################################ +# EKS Identity Provider +################################################################################ + +output "cluster_identity_providers" { + description = "Map of attribute maps for all EKS identity providers enabled" + value = module.eks.cluster_identity_providers +} + +################################################################################ +# CloudWatch Log Group +################################################################################ + +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_name +} + +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = module.eks.cloudwatch_log_group_arn +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.eks.fargate_profiles +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +output "eks_managed_node_groups" { + description = "Map of attribute maps for all EKS managed node groups created" + value = module.eks.eks_managed_node_groups +} + +output "eks_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by EKS managed node groups" + value = module.eks.eks_managed_node_groups_autoscaling_group_names +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +output "self_managed_node_groups" { + description = "Map of attribute maps for all self managed node groups created" + value = module.eks.self_managed_node_groups +} + +output "self_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by self-managed node groups" + value = module.eks.self_managed_node_groups_autoscaling_group_names +} diff --git a/tests/eks-fargate-profile/variables.tf b/tests/eks-fargate-profile/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf new file mode 100644 index 0000000000..5b36c201aa --- /dev/null +++ b/tests/eks-fargate-profile/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.2" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.79" + } + } +} diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md new file mode 100644 index 0000000000..341b73150e --- /dev/null +++ b/tests/eks-hybrid-nodes/README.md @@ -0,0 +1,65 @@ +# EKS Hybrid Node IAM Role + +## Usage + +To provision the provided configurations you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply --auto-approve +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [aws](#requirement\_aws) | >= 5.79 | +| [tls](#requirement\_tls) | >= 4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [tls](#provider\_tls) | >= 4.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [disabled\_eks\_hybrid\_node\_role](#module\_disabled\_eks\_hybrid\_node\_role) | ../../modules/hybrid-node-role | n/a | +| [eks\_hybrid\_node\_role](#module\_eks\_hybrid\_node\_role) | ../../modules/hybrid-node-role | n/a | +| [ira\_eks\_hybrid\_node\_role](#module\_ira\_eks\_hybrid\_node\_role) | ../../modules/hybrid-node-role | n/a | + +## Resources + +| Name | Type | +|------|------| +| [tls_private_key.example](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | +| [tls_self_signed_cert.example](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/self_signed_cert) | resource | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [arn](#output\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | +| [intermediate\_role\_arn](#output\_intermediate\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | +| [intermediate\_role\_name](#output\_intermediate\_role\_name) | The name of the node IAM role | +| [intermediate\_role\_unique\_id](#output\_intermediate\_role\_unique\_id) | Stable and unique string identifying the node IAM role | +| [ira\_arn](#output\_ira\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | +| [ira\_intermediate\_role\_arn](#output\_ira\_intermediate\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | +| [ira\_intermediate\_role\_name](#output\_ira\_intermediate\_role\_name) | The name of the node IAM role | +| [ira\_intermediate\_role\_unique\_id](#output\_ira\_intermediate\_role\_unique\_id) | Stable and unique string identifying the node IAM role | +| [ira\_name](#output\_ira\_name) | The name of the node IAM role | +| [ira\_unique\_id](#output\_ira\_unique\_id) | Stable and unique string identifying the node IAM role | +| [name](#output\_name) | The name of the node IAM role | +| [unique\_id](#output\_unique\_id) | Stable and unique string identifying the node IAM role | + diff --git a/tests/eks-hybrid-nodes/main.tf b/tests/eks-hybrid-nodes/main.tf new file mode 100644 index 0000000000..ec49725102 --- /dev/null +++ b/tests/eks-hybrid-nodes/main.tf @@ -0,0 +1,84 @@ +provider "aws" { + region = local.region +} + +locals { + name = "ex-${basename(path.cwd)}" + region = "us-west-2" + + tags = { + Test = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# Hybrid Node IAM Module +################################################################################ + +# Default (SSM) +module "eks_hybrid_node_role" { + source = "../../modules/hybrid-node-role" + + policy_statements = [ + { + actions = [ + "s3:Get*", + "s3:List*", + ] + resources = ["*"] + } + ] + + tags = local.tags +} + +# IAM Roles Anywhere +module "ira_eks_hybrid_node_role" { + source = "../../modules/hybrid-node-role" + + name = "${local.name}-ira" + + enable_ira = true + + ira_trust_anchor_source_type = "CERTIFICATE_BUNDLE" + ira_trust_anchor_x509_certificate_data = local.cert_data + + tags = local.tags +} + +module "disabled_eks_hybrid_node_role" { + source = "../../modules/hybrid-node-role" + + create = false +} + +################################################################################ +# Supporting Resources +################################################################################ + +resource "tls_private_key" "example" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "tls_self_signed_cert" "example" { + private_key_pem = tls_private_key.example.private_key_pem + + subject { + common_name = "Custom root" + organization = "ACME Examples, Inc" + } + + validity_period_hours = 17544 + is_ca_certificate = true + + allowed_uses = [ + "cert_signing", + ] +} + +locals { + cert_data = trimspace(replace(trimprefix(tls_self_signed_cert.example.cert_pem, "-----BEGIN CERTIFICATE-----"), "-----END CERTIFICATE-----", "")) +} diff --git a/tests/eks-hybrid-nodes/outputs.tf b/tests/eks-hybrid-nodes/outputs.tf new file mode 100644 index 0000000000..ffea7ccbed --- /dev/null +++ b/tests/eks-hybrid-nodes/outputs.tf @@ -0,0 +1,71 @@ +################################################################################ +# Default (SSM) - Node IAM Role +################################################################################ + +# Node IAM Role +output "name" { + description = "The name of the node IAM role" + value = module.eks_hybrid_node_role.name +} + +output "arn" { + description = "The Amazon Resource Name (ARN) specifying the node IAM role" + value = module.eks_hybrid_node_role.arn +} + +output "unique_id" { + description = "Stable and unique string identifying the node IAM role" + value = module.eks_hybrid_node_role.unique_id +} + +# Intermedaite IAM Role +output "intermediate_role_name" { + description = "The name of the node IAM role" + value = module.eks_hybrid_node_role.intermediate_role_name +} + +output "intermediate_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the node IAM role" + value = module.eks_hybrid_node_role.intermediate_role_arn +} + +output "intermediate_role_unique_id" { + description = "Stable and unique string identifying the node IAM role" + value = module.eks_hybrid_node_role.intermediate_role_unique_id +} + +################################################################################ +# IAM Roles Anywhere - Node IAM Role +################################################################################ + +# Node IAM Role +output "ira_name" { + description = "The name of the node IAM role" + value = module.ira_eks_hybrid_node_role.name +} + +output "ira_arn" { + description = "The Amazon Resource Name (ARN) specifying the node IAM role" + value = module.ira_eks_hybrid_node_role.arn +} + +output "ira_unique_id" { + description = "Stable and unique string identifying the node IAM role" + value = module.ira_eks_hybrid_node_role.unique_id +} + +# Intermedaite IAM Role +output "ira_intermediate_role_name" { + description = "The name of the node IAM role" + value = module.ira_eks_hybrid_node_role.intermediate_role_name +} + +output "ira_intermediate_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the node IAM role" + value = module.ira_eks_hybrid_node_role.intermediate_role_arn +} + +output "ira_intermediate_role_unique_id" { + description = "Stable and unique string identifying the node IAM role" + value = module.ira_eks_hybrid_node_role.intermediate_role_unique_id +} diff --git a/tests/eks-hybrid-nodes/variables.tf b/tests/eks-hybrid-nodes/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf new file mode 100644 index 0000000000..42232f11ed --- /dev/null +++ b/tests/eks-hybrid-nodes/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.3.2" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.79" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } + } +} diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index b1f4ab1398..6b5b02ab26 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.75 | +| [aws](#requirement\_aws) | >= 5.79 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.75 | +| [aws](#provider\_aws) | >= 5.79 | ## Modules @@ -68,8 +68,8 @@ No inputs. | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | | [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | -| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | -| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | Cluster IAM role ARN | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | Cluster IAM role name | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | @@ -89,6 +89,9 @@ No inputs. | [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | | [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | | [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | +| [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | EKS Auto node IAM role ARN | +| [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | EKS Auto node IAM role name | +| [node\_iam\_role\_unique\_id](#output\_node\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index 8a4d48c7f0..5d26a53fca 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -3,7 +3,14 @@ provider "aws" { } data "aws_caller_identity" "current" {} -data "aws_availability_zones" "available" {} + +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" diff --git a/tests/eks-managed-node-group/outputs.tf b/tests/eks-managed-node-group/outputs.tf index 9357464c29..9ed8c27220 100644 --- a/tests/eks-managed-node-group/outputs.tf +++ b/tests/eks-managed-node-group/outputs.tf @@ -142,12 +142,12 @@ output "cluster_tls_certificate_sha1_fingerprint" { ################################################################################ output "cluster_iam_role_name" { - description = "IAM role name of the EKS cluster" + description = "Cluster IAM role name" value = module.eks.cluster_iam_role_name } output "cluster_iam_role_arn" { - description = "IAM role ARN of the EKS cluster" + description = "Cluster IAM role ARN" value = module.eks.cluster_iam_role_arn } @@ -156,6 +156,25 @@ output "cluster_iam_role_unique_id" { value = module.eks.cluster_iam_role_unique_id } +################################################################################ +# EKS Auto Node IAM Role +################################################################################ + +output "node_iam_role_name" { + description = "EKS Auto node IAM role name" + value = module.eks.node_iam_role_name +} + +output "node_iam_role_arn" { + description = "EKS Auto node IAM role ARN" + value = module.eks.node_iam_role_arn +} + +output "node_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.node_iam_role_unique_id +} + ################################################################################ # EKS Addons ################################################################################ diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index 0099e6baaf..5b36c201aa 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } } } diff --git a/tests/fast-addons/README.md b/tests/fast-addons/README.md index 693784e79d..59a5cea926 100644 --- a/tests/fast-addons/README.md +++ b/tests/fast-addons/README.md @@ -22,13 +22,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.75 | +| [aws](#requirement\_aws) | >= 5.79 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.75 | +| [aws](#provider\_aws) | >= 5.79 | ## Modules diff --git a/tests/fast-addons/versions.tf b/tests/fast-addons/versions.tf index 0099e6baaf..5b36c201aa 100644 --- a/tests/fast-addons/versions.tf +++ b/tests/fast-addons/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 1587f7c177..437dfa851d 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.75 | +| [aws](#requirement\_aws) | >= 5.79 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.75 | +| [aws](#provider\_aws) | >= 5.79 | ## Modules @@ -64,8 +64,8 @@ No inputs. | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | | [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | -| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | -| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | +| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | Cluster IAM role ARN | +| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | Cluster IAM role name | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | @@ -85,6 +85,9 @@ No inputs. | [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | | [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | | [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | +| [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | EKS Auto node IAM role ARN | +| [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | EKS Auto node IAM role name | +| [node\_iam\_role\_unique\_id](#output\_node\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/tests/self-managed-node-group/main.tf b/tests/self-managed-node-group/main.tf index afe7aac9a1..f2181fdc75 100644 --- a/tests/self-managed-node-group/main.tf +++ b/tests/self-managed-node-group/main.tf @@ -3,7 +3,14 @@ provider "aws" { } data "aws_caller_identity" "current" {} -data "aws_availability_zones" "available" {} + +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" diff --git a/tests/self-managed-node-group/outputs.tf b/tests/self-managed-node-group/outputs.tf index 9357464c29..9ed8c27220 100644 --- a/tests/self-managed-node-group/outputs.tf +++ b/tests/self-managed-node-group/outputs.tf @@ -142,12 +142,12 @@ output "cluster_tls_certificate_sha1_fingerprint" { ################################################################################ output "cluster_iam_role_name" { - description = "IAM role name of the EKS cluster" + description = "Cluster IAM role name" value = module.eks.cluster_iam_role_name } output "cluster_iam_role_arn" { - description = "IAM role ARN of the EKS cluster" + description = "Cluster IAM role ARN" value = module.eks.cluster_iam_role_arn } @@ -156,6 +156,25 @@ output "cluster_iam_role_unique_id" { value = module.eks.cluster_iam_role_unique_id } +################################################################################ +# EKS Auto Node IAM Role +################################################################################ + +output "node_iam_role_name" { + description = "EKS Auto node IAM role name" + value = module.eks.node_iam_role_name +} + +output "node_iam_role_arn" { + description = "EKS Auto node IAM role ARN" + value = module.eks.node_iam_role_arn +} + +output "node_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.eks.node_iam_role_unique_id +} + ################################################################################ # EKS Addons ################################################################################ diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index 0099e6baaf..5b36c201aa 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } } } diff --git a/variables.tf b/variables.tf index edd0d938a6..e59ee3dc6e 100644 --- a/variables.tf +++ b/variables.tf @@ -44,12 +44,24 @@ variable "authentication_mode" { default = "API_AND_CONFIG_MAP" } +variable "cluster_compute_config" { + description = "Configuration block for the cluster compute configuration" + type = any + default = {} +} + variable "cluster_upgrade_policy" { description = "Configuration block for the cluster upgrade policy" type = any default = {} } +variable "cluster_remote_network_config" { + description = "Configuration block for the cluster remote network configuration" + type = any + default = {} +} + variable "cluster_zonal_shift_config" { description = "Configuration block for the cluster zonal shift" type = any @@ -434,7 +446,7 @@ variable "custom_oidc_thumbprints" { ################################################################################ variable "create_iam_role" { - description = "Determines whether a an IAM role is created or to use an existing IAM role" + description = "Determines whether an IAM role is created for the cluster" type = bool default = true } @@ -458,7 +470,7 @@ variable "iam_role_use_name_prefix" { } variable "iam_role_path" { - description = "Cluster IAM role path" + description = "The IAM role path" type = string default = null } @@ -481,6 +493,13 @@ variable "iam_role_additional_policies" { default = {} } +# TODO - will be removed in next breaking change; user can add the policy on their own when needed +variable "enable_security_groups_for_pods" { + description = "Determines whether to add the necessary IAM permission policy for security groups for pods" + type = bool + default = true +} + variable "iam_role_tags" { description = "A map of additional tags to add to the IAM role created" type = map(string) @@ -549,6 +568,70 @@ variable "cluster_identity_providers" { default = {} } +################################################################################ +# EKS Auto Node IAM Role +################################################################################ + +variable "create_node_iam_role" { + description = "Determines whether an EKS Auto node IAM role is created" + type = bool + default = true +} + +variable "node_iam_role_name" { + description = "Name to use on the EKS Auto node IAM role created" + type = string + default = null +} + +variable "node_iam_role_use_name_prefix" { + description = "Determines whether the EKS Auto node IAM role name (`node_iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "node_iam_role_path" { + description = "The EKS Auto node IAM role path" + type = string + default = null +} + +variable "node_iam_role_description" { + description = "Description of the EKS Auto node IAM role" + type = string + default = null +} + +variable "node_iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the EKS Auto node IAM role" + type = string + default = null +} + +variable "node_iam_role_additional_policies" { + description = "Additional policies to be added to the EKS Auto node IAM role" + type = map(string) + default = {} +} + +variable "node_iam_role_tags" { + description = "A map of additional tags to add to the EKS Auto node IAM role created" + type = map(string) + default = {} +} + +variable "enable_node_custom_tags_permissions" { + description = "Determines whether to enable permissions for custom tags for the EKS Auto node IAM role" + type = bool + default = true +} + +variable "node_iam_role_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + ################################################################################ # Fargate ################################################################################ diff --git a/versions.tf b/versions.tf index fc9dadd253..aaa00e9b19 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.75" + version = ">= 5.79" } tls = { source = "hashicorp/tls" From c63a57908d7b4d51895d3f8502d60daae758b761 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 4 Dec 2024 15:24:51 +0000 Subject: [PATCH 846/962] chore(release): version 20.31.0 [skip ci] ## [20.31.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.30.1...v20.31.0) (2024-12-04) ### Features * Add support for EKS Auto Mode and EKS Hybrid nodes ([#3225](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3225)) ([3b974d3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3b974d33ad79e142566dd7bcb4bf10472cc91899)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c7b9ec0ec..8ee2c8d588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.31.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.30.1...v20.31.0) (2024-12-04) + + +### Features + +* Add support for EKS Auto Mode and EKS Hybrid nodes ([#3225](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3225)) ([3b974d3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3b974d33ad79e142566dd7bcb4bf10472cc91899)) + ## [20.30.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.30.0...v20.30.1) (2024-11-26) From 6168518339ab4a8c50bfa25ee7e7c37d1f3597bd Mon Sep 17 00:00:00 2001 From: Lawrence Aiello Date: Sun, 8 Dec 2024 18:12:06 -0500 Subject: [PATCH 847/962] chore: Add `TODO` to turn off detailed monitoring by default at next breaking change (#3233) * chore: disable detailed monitoring by default * docs: change default input value * chore: add comment to change at next breaking change * docs: change default back --- modules/eks-managed-node-group/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index e0ee435785..b02f42cfcc 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -273,6 +273,7 @@ variable "metadata_options" { } } +# TODO - make this false by default at next breaking change variable "enable_monitoring" { description = "Enables/disables detailed monitoring" type = bool From e2846be8b110e59d36d6f868b74531a6d8ca4987 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 8 Dec 2024 22:42:11 -0600 Subject: [PATCH 848/962] fix: Create EKS Auto Mode role when Auto Mode is enabled, regardless of built-in node pool use (#3234) --- examples/karpenter/main.tf | 19 +++++-------------- main.tf | 23 ++++++++++++++--------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 9a6223e253..a7aee5ba7a 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -95,16 +95,6 @@ module "eks" { min_size = 2 max_size = 3 desired_size = 2 - - taints = { - # This Taint aims to keep just EKS Addons and Karpenter running on this MNG - # The pods that do not tolerate this taint should run on nodes created by Karpenter - addons = { - key = "CriticalAddonsOnly" - value = "true" - effect = "NO_SCHEDULE" - }, - } } } @@ -164,17 +154,18 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "1.0.6" + version = "1.1.0" wait = false values = [ <<-EOT - serviceAccount: - name: ${module.karpenter.service_account} + dnsPolicy: Default settings: clusterName: ${module.eks.cluster_name} clusterEndpoint: ${module.eks.cluster_endpoint} interruptionQueue: ${module.karpenter.queue_name} + webhook: + enabled: false EOT ] } @@ -226,7 +217,7 @@ resource "kubectl_manifest" "karpenter_node_pool" { values: ["nitro"] - key: "karpenter.k8s.aws/instance-generation" operator: Gt - values: ["2"] + values: ["5"] limits: cpu: 1000 disruption: diff --git a/main.tf b/main.tf index c2889b0dd4..6c15ff9ff9 100644 --- a/main.tf +++ b/main.tf @@ -1,26 +1,31 @@ -data "aws_partition" "current" {} -data "aws_caller_identity" "current" {} +data "aws_partition" "current" { + count = local.create ? 1 : 0 +} +data "aws_caller_identity" "current" { + count = local.create ? 1 : 0 +} data "aws_iam_session_context" "current" { + count = local.create ? 1 : 0 + # This data source provides information on the IAM source role of an STS assumed role # For non-role ARNs, this data source simply passes the ARN through issuer ARN # Ref https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2327#issuecomment-1355581682 # Ref https://github.com/hashicorp/terraform-provider-aws/issues/28381 - arn = data.aws_caller_identity.current.arn + arn = try(data.aws_caller_identity.current[0].arn, "") } locals { create = var.create && var.putin_khuylo - partition = data.aws_partition.current.partition + partition = try(data.aws_partition.current[0].partition, "") cluster_role = try(aws_iam_role.this[0].arn, var.iam_role_arn) create_outposts_local_cluster = length(var.outpost_config) > 0 enable_cluster_encryption_config = length(var.cluster_encryption_config) > 0 && !local.create_outposts_local_cluster - auto_mode_enabled = try(var.cluster_compute_config.enabled, false) - auto_mode_nodepools_enabled = length(try(var.cluster_compute_config.node_pools, [])) > 0 + auto_mode_enabled = try(var.cluster_compute_config.enabled, false) } ################################################################################ @@ -218,7 +223,7 @@ locals { # better controlled by users through Terraform bootstrap_cluster_creator_admin_permissions = { cluster_creator = { - principal_arn = data.aws_iam_session_context.current.issuer_arn + principal_arn = try(data.aws_iam_session_context.current[0].issuer_arn, "") type = "STANDARD" policy_associations = { @@ -307,7 +312,7 @@ module "kms" { # Policy enable_default_policy = var.kms_key_enable_default_policy key_owners = var.kms_key_owners - key_administrators = coalescelist(var.kms_key_administrators, [data.aws_iam_session_context.current.issuer_arn]) + key_administrators = coalescelist(var.kms_key_administrators, [try(data.aws_iam_session_context.current[0].issuer_arn, "")]) key_users = concat([local.cluster_role], var.kms_key_users) key_service_users = var.kms_key_service_users source_policy_documents = var.kms_key_source_policy_documents @@ -689,7 +694,7 @@ resource "aws_eks_identity_provider_config" "this" { ################################################################################ locals { - create_node_iam_role = local.create && var.create_node_iam_role && local.auto_mode_nodepools_enabled + create_node_iam_role = local.create && var.create_node_iam_role && local.auto_mode_enabled node_iam_role_name = coalesce(var.node_iam_role_name, "${var.cluster_name}-eks-auto") create_node_iam_role_custom_policy = local.create_node_iam_role && (var.enable_node_custom_tags_permissions || length(var.node_iam_role_policy_statements) > 0) From 5583604ffad307e3e894434248a35293fca51e9e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 9 Dec 2024 04:42:36 +0000 Subject: [PATCH 849/962] chore(release): version 20.31.1 [skip ci] ## [20.31.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.0...v20.31.1) (2024-12-09) ### Bug Fixes * Create EKS Auto Mode role when Auto Mode is enabled, regardless of built-in node pool use ([#3234](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3234)) ([e2846be](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e2846be8b110e59d36d6f868b74531a6d8ca4987)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ee2c8d588..bbf4a04982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.31.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.0...v20.31.1) (2024-12-09) + + +### Bug Fixes + +* Create EKS Auto Mode role when Auto Mode is enabled, regardless of built-in node pool use ([#3234](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3234)) ([e2846be](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e2846be8b110e59d36d6f868b74531a6d8ca4987)) + ## [20.31.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.30.1...v20.31.0) (2024-12-04) From ce34f1db3f7824167d9a766e6c90dee3a6dcf1c3 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 12 Dec 2024 07:14:15 -0600 Subject: [PATCH 850/962] fix: Avoid trying to attach the node role when Auto Mode nodepools are not specified (#3239) --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 6c15ff9ff9..82b2fb190d 100644 --- a/main.tf +++ b/main.tf @@ -58,7 +58,7 @@ resource "aws_eks_cluster" "this" { content { enabled = try(compute_config.value.enabled, null) node_pools = local.auto_mode_enabled ? try(compute_config.value.node_pools, []) : null - node_role_arn = local.auto_mode_enabled ? try(compute_config.value.node_role_arn, aws_iam_role.eks_auto[0].arn, null) : null + node_role_arn = local.auto_mode_enabled && length(try(compute_config.value.node_pools, [])) > 0 ? try(compute_config.value.node_role_arn, aws_iam_role.eks_auto[0].arn, null) : null } } From ad88d72bdd2fe2ce2b9885548e7127c583d0b18d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 12 Dec 2024 13:14:42 +0000 Subject: [PATCH 851/962] chore(release): version 20.31.2 [skip ci] ## [20.31.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.1...v20.31.2) (2024-12-12) ### Bug Fixes * Avoid trying to attach the node role when Auto Mode nodepools are not specified ([#3239](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3239)) ([ce34f1d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ce34f1db3f7824167d9a766e6c90dee3a6dcf1c3)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbf4a04982..19404cceae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.31.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.1...v20.31.2) (2024-12-12) + + +### Bug Fixes + +* Avoid trying to attach the node role when Auto Mode nodepools are not specified ([#3239](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3239)) ([ce34f1d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ce34f1db3f7824167d9a766e6c90dee3a6dcf1c3)) + ## [20.31.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.0...v20.31.1) (2024-12-09) From 012e51c05551da48a7f380d4a7b75880b0c24fe1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 12 Dec 2024 14:49:59 -0600 Subject: [PATCH 852/962] fix: Update min provider version to remediate cluster replacement when enabling EKS Auto Mode (#3240) --- README.md | 4 ++-- examples/eks-auto-mode/README.md | 4 ++-- examples/eks-auto-mode/versions.tf | 2 +- examples/eks-hybrid-nodes/README.md | 11 +++++++---- examples/eks-hybrid-nodes/versions.tf | 2 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 6 +++--- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/hybrid-node-role/README.md | 4 ++-- modules/hybrid-node-role/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-fargate-profile/README.md | 4 ++-- tests/eks-fargate-profile/versions.tf | 2 +- tests/eks-hybrid-nodes/README.md | 2 +- tests/eks-hybrid-nodes/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/versions.tf | 2 +- tests/fast-addons/README.md | 4 ++-- tests/fast-addons/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- versions.tf | 2 +- 30 files changed, 49 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 8f7cc2727b..bd1b4ddc08 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -326,7 +326,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index d12605de93..ee2095bf73 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -25,13 +25,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | ## Modules diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index df32f7d8ac..058c49386f 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -1,4 +1,7 @@ -# EKS Hybrid Node IAM Role +# EKS Hybrid Nodes + +> [!CAUTION] +> EC2 instances are not supported with EKS Hybrid Nodes. This example is provided for reference only in lieu of users having to provision a VM in their own environment. ## Usage @@ -23,7 +26,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | | [helm](#requirement\_helm) | >= 2.16 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | @@ -33,8 +36,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | -| [aws.remote](#provider\_aws.remote) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | +| [aws.remote](#provider\_aws.remote) | >= 5.81 | | [helm](#provider\_helm) | >= 2.16 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index 6cd8533c17..acdd4737dd 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } helm = { source = "hashicorp/helm" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 87a7d8ae78..973bced534 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -89,7 +89,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -97,8 +97,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.81 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 5d3e660ea4..6c8b905a55 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 0936b8f790..5097c7573f 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 0d47f4697a..1db25fbf57 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index bd93c4ba1d..3f893571e3 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -75,13 +75,13 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | ## Modules diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 891d1635df..8ef4729463 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 654e223b1e..b342044094 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index 9035c4efbf..afe94054f9 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | ## Modules diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index 341b73150e..6bc9e7d1eb 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -18,7 +18,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index 42232f11ed..19bf4edb5e 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 6b5b02ab26..1434e52b62 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/tests/fast-addons/README.md b/tests/fast-addons/README.md index 59a5cea926..ae78a4ee73 100644 --- a/tests/fast-addons/README.md +++ b/tests/fast-addons/README.md @@ -22,13 +22,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | ## Modules diff --git a/tests/fast-addons/versions.tf b/tests/fast-addons/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/tests/fast-addons/versions.tf +++ b/tests/fast-addons/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 437dfa851d..3c26d9e435 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.79 | +| [aws](#requirement\_aws) | >= 5.81 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.79 | +| [aws](#provider\_aws) | >= 5.81 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index 5b36c201aa..e31fee1f34 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } } } diff --git a/versions.tf b/versions.tf index aaa00e9b19..33405ee6ba 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.79" + version = ">= 5.81" } tls = { source = "hashicorp/tls" From bba5a75ee1310b220cd2b073b2b9cda48543aefc Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 12 Dec 2024 20:50:27 +0000 Subject: [PATCH 853/962] chore(release): version 20.31.3 [skip ci] ## [20.31.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.2...v20.31.3) (2024-12-12) ### Bug Fixes * Update min provider version to remediate cluster replacement when enabling EKS Auto Mode ([#3240](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3240)) ([012e51c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/012e51c05551da48a7f380d4a7b75880b0c24fe1)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19404cceae..3367d354cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.31.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.2...v20.31.3) (2024-12-12) + + +### Bug Fixes + +* Update min provider version to remediate cluster replacement when enabling EKS Auto Mode ([#3240](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3240)) ([012e51c](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/012e51c05551da48a7f380d4a7b75880b0c24fe1)) + ## [20.31.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.1...v20.31.2) (2024-12-12) From a07013a1f4d4d56b56eb2e6265a6f38041a4540b Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 14 Dec 2024 07:28:52 -0600 Subject: [PATCH 854/962] fix: Auto Mode custom tag policy should apply to cluster role, not node role (#3242) --- README.md | 9 +- main.tf | 310 +++++++++++++++++++++++++-------------------------- variables.tf | 18 +-- 3 files changed, 164 insertions(+), 173 deletions(-) diff --git a/README.md b/README.md index bd1b4ddc08..ca83185353 100644 --- a/README.md +++ b/README.md @@ -354,14 +354,14 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_iam_openid_connect_provider.oidc_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource | | [aws_iam_policy.cluster_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.eks_auto_custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.eks_auto](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cluster_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.eks_auto](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.eks_auto_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.eks_auto_custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_security_group.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | @@ -372,7 +372,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.eks_auto_custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.node_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_session_context.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_session_context) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -435,11 +435,11 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed node group(s), self-managed node group(s), Fargate profile(s)) | `string` | `"30s"` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | +| [enable\_auto\_mode\_custom\_tags](#input\_enable\_auto\_mode\_custom\_tags) | Determines whether to enable permissions for custom tags resources created by EKS Auto Mode | `bool` | `true` | no | | [enable\_cluster\_creator\_admin\_permissions](#input\_enable\_cluster\_creator\_admin\_permissions) | Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry | `bool` | `false` | no | | [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | | [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled | `bool` | `true` | no | -| [enable\_node\_custom\_tags\_permissions](#input\_enable\_node\_custom\_tags\_permissions) | Determines whether to enable permissions for custom tags for the EKS Auto node IAM role | `bool` | `true` | no | | [enable\_security\_groups\_for\_pods](#input\_enable\_security\_groups\_for\_pods) | Determines whether to add the necessary IAM permission policy for security groups for pods | `bool` | `true` | no | | [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no | | [fargate\_profiles](#input\_fargate\_profiles) | Map of Fargate Profile definitions to create | `any` | `{}` | no | @@ -467,7 +467,6 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [node\_iam\_role\_name](#input\_node\_iam\_role\_name) | Name to use on the EKS Auto node IAM role created | `string` | `null` | no | | [node\_iam\_role\_path](#input\_node\_iam\_role\_path) | The EKS Auto node IAM role path | `string` | `null` | no | | [node\_iam\_role\_permissions\_boundary](#input\_node\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the EKS Auto node IAM role | `string` | `null` | no | -| [node\_iam\_role\_policy\_statements](#input\_node\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | | [node\_iam\_role\_tags](#input\_node\_iam\_role\_tags) | A map of additional tags to add to the EKS Auto node IAM role created | `map(string)` | `{}` | no | | [node\_iam\_role\_use\_name\_prefix](#input\_node\_iam\_role\_use\_name\_prefix) | Determines whether the EKS Auto node IAM role name (`node_iam_role_name`) is used as a prefix | `bool` | `true` | no | | [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no | diff --git a/main.tf b/main.tf index 82b2fb190d..c2bb88f6d0 100644 --- a/main.tf +++ b/main.tf @@ -563,6 +563,160 @@ resource "aws_iam_policy" "cluster_encryption" { tags = merge(var.tags, var.cluster_encryption_policy_tags) } +data "aws_iam_policy_document" "custom" { + count = local.create_iam_role && var.enable_auto_mode_custom_tags ? 1 : 0 + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "Compute" + actions = [ + "ec2:CreateFleet", + "ec2:RunInstances", + "ec2:CreateLaunchTemplate", + ] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/eks:kubernetes-node-class-name" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/eks:kubernetes-node-pool-name" + values = ["*"] + } + } + } + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "Storage" + actions = [ + "ec2:CreateVolume", + "ec2:CreateSnapshot", + ] + resources = [ + "arn:${local.partition}:ec2:*:*:volume/*", + "arn:${local.partition}:ec2:*:*:snapshot/*", + ] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "Networking" + actions = ["ec2:CreateNetworkInterface"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:kubernetes-cni-node-name" + values = ["*"] + } + } + } + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "LoadBalancer" + actions = [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateRule", + "ec2:CreateSecurityGroup", + ] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "ShieldProtection" + actions = ["shield:CreateProtection"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "ShieldTagResource" + actions = ["shield:TagResource"] + resources = ["arn:${local.partition}:shield::*:protection/*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } +} + +resource "aws_iam_policy" "custom" { + count = local.create_iam_role && var.enable_auto_mode_custom_tags ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + policy = data.aws_iam_policy_document.custom[0].json + + tags = merge(var.tags, var.iam_role_tags) +} + +resource "aws_iam_role_policy_attachment" "custom" { + count = local.create_iam_role && var.enable_auto_mode_custom_tags ? 1 : 0 + + policy_arn = aws_iam_policy.custom[0].arn + role = aws_iam_role.this[0].name +} + ################################################################################ # EKS Addons ################################################################################ @@ -696,8 +850,6 @@ resource "aws_eks_identity_provider_config" "this" { locals { create_node_iam_role = local.create && var.create_node_iam_role && local.auto_mode_enabled node_iam_role_name = coalesce(var.node_iam_role_name, "${var.cluster_name}-eks-auto") - - create_node_iam_role_custom_policy = local.create_node_iam_role && (var.enable_node_custom_tags_permissions || length(var.node_iam_role_policy_statements) > 0) } data "aws_iam_policy_document" "node_assume_role_policy" { @@ -749,157 +901,3 @@ resource "aws_iam_role_policy_attachment" "eks_auto_additional" { policy_arn = each.value role = aws_iam_role.eks_auto[0].name } - -resource "aws_iam_role_policy_attachment" "eks_auto_custom" { - count = local.create_node_iam_role_custom_policy ? 1 : 0 - - policy_arn = aws_iam_policy.eks_auto_custom[0].arn - role = aws_iam_role.eks_auto[0].name -} - -data "aws_iam_policy_document" "eks_auto_custom" { - count = local.create_node_iam_role_custom_policy ? 1 : 0 - - dynamic "statement" { - for_each = var.enable_node_custom_tags_permissions ? [1] : [] - - content { - sid = "Compute" - actions = [ - "ec2:CreateFleet", - "ec2:RunInstances", - "ec2:CreateLaunchTemplate", - ] - resources = ["*"] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/eks:eks-cluster-name" - values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] - } - - condition { - test = "StringLike" - variable = "aws:RequestTag/eks:kubernetes-node-class-name" - values = ["*"] - } - - condition { - test = "StringLike" - variable = "aws:RequestTag/eks:kubernetes-node-pool-name" - values = ["*"] - } - } - } - - dynamic "statement" { - for_each = var.enable_node_custom_tags_permissions ? [1] : [] - - content { - sid = "Storage" - actions = [ - "ec2:CreateVolume", - "ec2:CreateSnapshot", - ] - resources = [ - "arn:${local.partition}:ec2:*:*:volume/*", - "arn:${local.partition}:ec2:*:*:snapshot/*", - ] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/eks:eks-cluster-name" - values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] - } - } - } - - dynamic "statement" { - for_each = var.enable_node_custom_tags_permissions ? [1] : [] - - content { - sid = "Networking" - actions = ["ec2:CreateNetworkInterface"] - resources = ["*"] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/eks:eks-cluster-name" - values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] - } - - condition { - test = "StringEquals" - variable = "aws:RequestTag/eks:kubernetes-cni-node-name" - values = ["*"] - } - } - } - - dynamic "statement" { - for_each = var.enable_node_custom_tags_permissions ? [1] : [] - - content { - sid = "LoadBalancer" - actions = [ - "elasticloadbalancing:CreateLoadBalancer", - "elasticloadbalancing:CreateTargetGroup", - "elasticloadbalancing:CreateListener", - "elasticloadbalancing:CreateRule", - "ec2:CreateSecurityGroup", - ] - resources = ["*"] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/eks:eks-cluster-name" - values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] - } - } - } - - dynamic "statement" { - for_each = var.enable_node_custom_tags_permissions ? [1] : [] - - content { - sid = "ShieldProtection" - actions = ["shield:CreateProtection"] - resources = ["*"] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/eks:eks-cluster-name" - values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] - } - } - } - - dynamic "statement" { - for_each = var.enable_node_custom_tags_permissions ? [1] : [] - - content { - sid = "ShieldTagResource" - actions = ["shield:TagResource"] - resources = ["arn:${local.partition}:shield::*:protection/*"] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/eks:eks-cluster-name" - values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] - } - } - } -} - -resource "aws_iam_policy" "eks_auto_custom" { - count = local.create_node_iam_role_custom_policy ? 1 : 0 - - name = var.node_iam_role_use_name_prefix ? null : local.node_iam_role_name - name_prefix = var.node_iam_role_use_name_prefix ? "${local.node_iam_role_name}-" : null - path = var.node_iam_role_path - description = var.node_iam_role_description - - policy = data.aws_iam_policy_document.eks_auto_custom[0].json - - tags = merge(var.tags, var.node_iam_role_tags) -} diff --git a/variables.tf b/variables.tf index e59ee3dc6e..855c2133ec 100644 --- a/variables.tf +++ b/variables.tf @@ -542,6 +542,12 @@ variable "dataplane_wait_duration" { default = "30s" } +variable "enable_auto_mode_custom_tags" { + description = "Determines whether to enable permissions for custom tags resources created by EKS Auto Mode" + type = bool + default = true +} + ################################################################################ # EKS Addons ################################################################################ @@ -620,18 +626,6 @@ variable "node_iam_role_tags" { default = {} } -variable "enable_node_custom_tags_permissions" { - description = "Determines whether to enable permissions for custom tags for the EKS Auto node IAM role" - type = bool - default = true -} - -variable "node_iam_role_policy_statements" { - description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" - type = any - default = [] -} - ################################################################################ # Fargate ################################################################################ From ed43ce7ec6004512ec22adfd3302a2cbdb3b6f23 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 14 Dec 2024 13:29:17 +0000 Subject: [PATCH 855/962] chore(release): version 20.31.4 [skip ci] ## [20.31.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.3...v20.31.4) (2024-12-14) ### Bug Fixes * Auto Mode custom tag policy should apply to cluster role, not node role ([#3242](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3242)) ([a07013a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a07013a1f4d4d56b56eb2e6265a6f38041a4540b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3367d354cc..6fd330ef95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.31.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.3...v20.31.4) (2024-12-14) + + +### Bug Fixes + +* Auto Mode custom tag policy should apply to cluster role, not node role ([#3242](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3242)) ([a07013a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/a07013a1f4d4d56b56eb2e6265a6f38041a4540b)) + ## [20.31.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.2...v20.31.3) (2024-12-12) From 29d2a8c55195f3a14033709a7461c8404b946f1e Mon Sep 17 00:00:00 2001 From: Doruk Ozturk Date: Thu, 19 Dec 2024 12:06:36 -0500 Subject: [PATCH 856/962] docs: Correct EKS Hybrid Nodes execution steps to apply successfully (#3248) Update README.md --- examples/eks-hybrid-nodes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index 058c49386f..5ca82dce20 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -12,7 +12,7 @@ To provision the provided configurations you need to execute: ```bash terraform init -terraform apply -target=module.remote_node_vpc -target=local_file.key_pem --auto-approve +terraform apply -target=module.remote_node_vpc -target=local_file.key_pem -target=module.key_pair --auto-approve cd ami && packer build -var 'ssh_keypair_name=hybrid-node' -var 'ssh_private_key_file=../key.pem' . && cd - terraform apply --auto-approve ./join.sh From 2a6a57a9bb1c6563608985bbdbfb7f47eec971df Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 20 Dec 2024 12:36:19 -0600 Subject: [PATCH 857/962] fix: Correct Auto Mode disable (#3253) --- main.tf | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/main.tf b/main.tf index c2bb88f6d0..89a5efd1a0 100644 --- a/main.tf +++ b/main.tf @@ -56,7 +56,7 @@ resource "aws_eks_cluster" "this" { for_each = length(var.cluster_compute_config) > 0 ? [var.cluster_compute_config] : [] content { - enabled = try(compute_config.value.enabled, null) + enabled = local.auto_mode_enabled node_pools = local.auto_mode_enabled ? try(compute_config.value.node_pools, []) : null node_role_arn = local.auto_mode_enabled && length(try(compute_config.value.node_pools, [])) > 0 ? try(compute_config.value.node_role_arn, aws_iam_role.eks_auto[0].arn, null) : null } @@ -75,12 +75,9 @@ resource "aws_eks_cluster" "this" { for_each = local.create_outposts_local_cluster ? [] : [1] content { - dynamic "elastic_load_balancing" { - for_each = local.auto_mode_enabled ? [1] : [] + elastic_load_balancing { - content { - enabled = local.auto_mode_enabled - } + enabled = local.auto_mode_enabled } ip_family = var.cluster_ip_family @@ -133,13 +130,9 @@ resource "aws_eks_cluster" "this" { } } - dynamic "storage_config" { - for_each = local.auto_mode_enabled ? [1] : [] - - content { - block_storage { - enabled = local.auto_mode_enabled - } + storage_config { + block_storage { + enabled = local.auto_mode_enabled } } From 791b9054a12fbf2a3041585feceae52eadeacac4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 20 Dec 2024 18:36:47 +0000 Subject: [PATCH 858/962] chore(release): version 20.31.5 [skip ci] ## [20.31.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.4...v20.31.5) (2024-12-20) ### Bug Fixes * Correct Auto Mode disable ([#3253](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3253)) ([2a6a57a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/2a6a57a9bb1c6563608985bbdbfb7f47eec971df)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fd330ef95..b62cb191f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.31.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.4...v20.31.5) (2024-12-20) + + +### Bug Fixes + +* Correct Auto Mode disable ([#3253](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3253)) ([2a6a57a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/2a6a57a9bb1c6563608985bbdbfb7f47eec971df)) + ## [20.31.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.3...v20.31.4) (2024-12-14) From 9fa75c0dedba2c89e9c316a597f23a0129c34d07 Mon Sep 17 00:00:00 2001 From: Yaro Date: Fri, 20 Dec 2024 21:17:45 +0200 Subject: [PATCH 859/962] chore: Remove `kubectl` provider from Karpenter example (#3251) * Change kubectl provider * chore: Remove `kubectl` provider --------- Co-authored-by: Bryant Biggs --- examples/karpenter/README.md | 60 +++++++------- examples/karpenter/inflate.yaml | 21 +++++ examples/karpenter/karpenter.yaml | 47 +++++++++++ examples/karpenter/main.tf | 130 +++--------------------------- examples/karpenter/versions.tf | 4 - 5 files changed, 111 insertions(+), 151 deletions(-) create mode 100644 examples/karpenter/inflate.yaml create mode 100644 examples/karpenter/karpenter.yaml diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 973bced534..473b1b49c6 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -18,8 +18,11 @@ Once the cluster is up and running, you can check that Karpenter is functioning # First, make sure you have updated your local kubeconfig aws eks --region eu-west-1 update-kubeconfig --name ex-karpenter -# Second, scale the example deployment -kubectl scale deployment inflate --replicas 5 +# Second, deploy the Karpenter NodeClass/NodePool +kubectl apply -f karpenter.yaml + +# Second, deploy the example deployment +kubectl apply -f inflate.yaml # You can watch Karpenter's controller logs with kubectl logs -f -n kube-system -l app.kubernetes.io/name=karpenter -c controller @@ -32,10 +35,10 @@ kubectl get nodes -L karpenter.sh/registered ``` ```text -NAME STATUS ROLES AGE VERSION REGISTERED -ip-10-0-16-155.eu-west-1.compute.internal Ready 100s v1.29.3-eks-ae9a62a true -ip-10-0-3-23.eu-west-1.compute.internal Ready 6m1s v1.29.3-eks-ae9a62a -ip-10-0-41-2.eu-west-1.compute.internal Ready 6m3s v1.29.3-eks-ae9a62a +NAME STATUS ROLES AGE VERSION REGISTERED +ip-10-0-13-51.eu-west-1.compute.internal Ready 29s v1.31.1-eks-1b3e656 true +ip-10-0-41-242.eu-west-1.compute.internal Ready 35m v1.31.1-eks-1b3e656 +ip-10-0-8-151.eu-west-1.compute.internal Ready 35m v1.31.1-eks-1b3e656 ``` ```sh @@ -44,24 +47,27 @@ kubectl get pods -A -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName ```text NAME NODE -inflate-75d744d4c6-nqwz8 ip-10-0-16-155.eu-west-1.compute.internal -inflate-75d744d4c6-nrqnn ip-10-0-16-155.eu-west-1.compute.internal -inflate-75d744d4c6-sp4dx ip-10-0-16-155.eu-west-1.compute.internal -inflate-75d744d4c6-xqzd9 ip-10-0-16-155.eu-west-1.compute.internal -inflate-75d744d4c6-xr6p5 ip-10-0-16-155.eu-west-1.compute.internal -aws-node-mnn7r ip-10-0-3-23.eu-west-1.compute.internal -aws-node-rkmvm ip-10-0-16-155.eu-west-1.compute.internal -aws-node-s4slh ip-10-0-41-2.eu-west-1.compute.internal -coredns-68bd859788-7rcfq ip-10-0-3-23.eu-west-1.compute.internal -coredns-68bd859788-l78hw ip-10-0-41-2.eu-west-1.compute.internal -eks-pod-identity-agent-gbx8l ip-10-0-41-2.eu-west-1.compute.internal -eks-pod-identity-agent-s7vt7 ip-10-0-16-155.eu-west-1.compute.internal -eks-pod-identity-agent-xwgqw ip-10-0-3-23.eu-west-1.compute.internal -karpenter-79f59bdfdc-9q5ff ip-10-0-41-2.eu-west-1.compute.internal -karpenter-79f59bdfdc-cxvhr ip-10-0-3-23.eu-west-1.compute.internal -kube-proxy-7crbl ip-10-0-41-2.eu-west-1.compute.internal -kube-proxy-jtzds ip-10-0-16-155.eu-west-1.compute.internal -kube-proxy-sm42c ip-10-0-3-23.eu-west-1.compute.internal +inflate-67cd5bb766-hvqfn ip-10-0-13-51.eu-west-1.compute.internal +inflate-67cd5bb766-jnsdp ip-10-0-13-51.eu-west-1.compute.internal +inflate-67cd5bb766-k4gwf ip-10-0-41-242.eu-west-1.compute.internal +inflate-67cd5bb766-m49f6 ip-10-0-13-51.eu-west-1.compute.internal +inflate-67cd5bb766-pgzx9 ip-10-0-8-151.eu-west-1.compute.internal +aws-node-58m4v ip-10-0-3-57.eu-west-1.compute.internal +aws-node-pj2gc ip-10-0-8-151.eu-west-1.compute.internal +aws-node-thffj ip-10-0-41-242.eu-west-1.compute.internal +aws-node-vh66d ip-10-0-13-51.eu-west-1.compute.internal +coredns-844dbb9f6f-9g9lg ip-10-0-41-242.eu-west-1.compute.internal +coredns-844dbb9f6f-fmzfq ip-10-0-41-242.eu-west-1.compute.internal +eks-pod-identity-agent-jr2ns ip-10-0-8-151.eu-west-1.compute.internal +eks-pod-identity-agent-mpjkq ip-10-0-13-51.eu-west-1.compute.internal +eks-pod-identity-agent-q4tjc ip-10-0-3-57.eu-west-1.compute.internal +eks-pod-identity-agent-zzfdj ip-10-0-41-242.eu-west-1.compute.internal +karpenter-5b8965dc9b-rx9bx ip-10-0-8-151.eu-west-1.compute.internal +karpenter-5b8965dc9b-xrfnx ip-10-0-41-242.eu-west-1.compute.internal +kube-proxy-2xf42 ip-10-0-41-242.eu-west-1.compute.internal +kube-proxy-kbfc8 ip-10-0-8-151.eu-west-1.compute.internal +kube-proxy-kt8zn ip-10-0-13-51.eu-west-1.compute.internal +kube-proxy-sl6bz ip-10-0-3-57.eu-west-1.compute.internal ``` ### Tear Down & Clean-Up @@ -72,7 +78,6 @@ Because Karpenter manages the state of node resources outside of Terraform, Karp ```bash kubectl delete deployment inflate -kubectl delete node -l karpenter.sh/provisioner-name=default ``` 2. Remove the resources created by Terraform @@ -91,7 +96,6 @@ Note that this example may create resources which cost money. Run `terraform des | [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.81 | | [helm](#requirement\_helm) | >= 2.7 | -| [kubectl](#requirement\_kubectl) | >= 2.0 | ## Providers @@ -100,7 +104,6 @@ Note that this example may create resources which cost money. Run `terraform des | [aws](#provider\_aws) | >= 5.81 | | [aws.virginia](#provider\_aws.virginia) | >= 5.81 | | [helm](#provider\_helm) | >= 2.7 | -| [kubectl](#provider\_kubectl) | >= 2.0 | ## Modules @@ -116,9 +119,6 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_node_class](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | diff --git a/examples/karpenter/inflate.yaml b/examples/karpenter/inflate.yaml new file mode 100644 index 0000000000..b70cfd8509 --- /dev/null +++ b/examples/karpenter/inflate.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 5 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 diff --git a/examples/karpenter/karpenter.yaml b/examples/karpenter/karpenter.yaml new file mode 100644 index 0000000000..3d3107c06f --- /dev/null +++ b/examples/karpenter/karpenter.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiSelectorTerms: + - alias: bottlerocket@latest + role: ex-karpenter + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: ex-karpenter + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: ex-karpenter + tags: + karpenter.sh/discovery: ex-karpenter +--- +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + limits: + cpu: 1000 + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 30s diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index a7aee5ba7a..9961063c95 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -21,20 +21,6 @@ provider "helm" { } } -provider "kubectl" { - apply_retry_count = 5 - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - load_config_file = false - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - data "aws_availability_zones" "available" { # Exclude local zones filter { @@ -89,21 +75,20 @@ module "eks" { eks_managed_node_groups = { karpenter = { - ami_type = "AL2023_x86_64_STANDARD" + ami_type = "BOTTLEROCKET_x86_64" instance_types = ["m5.large"] min_size = 2 max_size = 3 desired_size = 2 + + labels = { + # Used to ensure Karpenter runs on nodes that it does not manage + "karpenter.sh/controller" = "true" + } } } - # cluster_tags = merge(local.tags, { - # NOTE - only use this option if you are using "attach_cluster_primary_security_group" - # and you know what you're doing. In this case, you can remove the "node_security_group_tags" below. - # "karpenter.sh/discovery" = local.name - # }) - node_security_group_tags = merge(local.tags, { # NOTE - if creating multiple security groups with this module, only tag the # security group that Karpenter should utilize with the following tag @@ -121,11 +106,12 @@ module "eks" { module "karpenter" { source = "../../modules/karpenter" - cluster_name = module.eks.cluster_name - + cluster_name = module.eks.cluster_name enable_v1_permissions = true - enable_pod_identity = true + # Name needs to match role name passed to the EC2NodeClass + node_iam_role_use_name_prefix = false + node_iam_role_name = local.name create_pod_identity_association = true # Used to attach additional IAM policies to the Karpenter node IAM role @@ -154,11 +140,13 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "1.1.0" + version = "1.1.1" wait = false values = [ <<-EOT + nodeSelector: + karpenter.sh/controller: 'true' dnsPolicy: Default settings: clusterName: ${module.eks.cluster_name} @@ -170,98 +158,6 @@ resource "helm_release" "karpenter" { ] } -resource "kubectl_manifest" "karpenter_node_class" { - yaml_body = <<-YAML - apiVersion: karpenter.k8s.aws/v1beta1 - kind: EC2NodeClass - metadata: - name: default - spec: - amiFamily: AL2023 - role: ${module.karpenter.node_iam_role_name} - subnetSelectorTerms: - - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} - securityGroupSelectorTerms: - - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} - YAML - - depends_on = [ - helm_release.karpenter - ] -} - -resource "kubectl_manifest" "karpenter_node_pool" { - yaml_body = <<-YAML - apiVersion: karpenter.sh/v1beta1 - kind: NodePool - metadata: - name: default - spec: - template: - spec: - nodeClassRef: - name: default - requirements: - - key: "karpenter.k8s.aws/instance-category" - operator: In - values: ["c", "m", "r"] - - key: "karpenter.k8s.aws/instance-cpu" - operator: In - values: ["4", "8", "16", "32"] - - key: "karpenter.k8s.aws/instance-hypervisor" - operator: In - values: ["nitro"] - - key: "karpenter.k8s.aws/instance-generation" - operator: Gt - values: ["5"] - limits: - cpu: 1000 - disruption: - consolidationPolicy: WhenEmpty - consolidateAfter: 30s - YAML - - depends_on = [ - kubectl_manifest.karpenter_node_class - ] -} - -# Example deployment using the [pause image](https://www.ianlewis.org/en/almighty-pause-container) -# and starts with zero replicas -resource "kubectl_manifest" "karpenter_example_deployment" { - yaml_body = <<-YAML - apiVersion: apps/v1 - kind: Deployment - metadata: - name: inflate - spec: - replicas: 0 - selector: - matchLabels: - app: inflate - template: - metadata: - labels: - app: inflate - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: inflate - image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 - resources: - requests: - cpu: 1 - YAML - - depends_on = [ - helm_release.karpenter - ] -} - ################################################################################ # Supporting Resources ################################################################################ diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 6c8b905a55..08067c5339 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -10,9 +10,5 @@ terraform { source = "hashicorp/helm" version = ">= 2.7" } - kubectl = { - source = "alekc/kubectl" - version = ">= 2.0" - } } } From 1ac67b8a60e336285c4dca03e550dfc78d64acce Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 20 Dec 2024 16:26:52 -0600 Subject: [PATCH 860/962] fix: Revert changes to disabling auto mode #3253 (#3255) --- main.tf | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/main.tf b/main.tf index 89a5efd1a0..3c955d54d0 100644 --- a/main.tf +++ b/main.tf @@ -75,9 +75,12 @@ resource "aws_eks_cluster" "this" { for_each = local.create_outposts_local_cluster ? [] : [1] content { - elastic_load_balancing { + dynamic "elastic_load_balancing" { + for_each = local.auto_mode_enabled ? [1] : [] - enabled = local.auto_mode_enabled + content { + enabled = local.auto_mode_enabled + } } ip_family = var.cluster_ip_family @@ -130,9 +133,13 @@ resource "aws_eks_cluster" "this" { } } - storage_config { - block_storage { - enabled = local.auto_mode_enabled + dynamic "storage_config" { + for_each = local.auto_mode_enabled ? [1] : [] + + content { + block_storage { + enabled = local.auto_mode_enabled + } } } From a713f6f464eb579a39918f60f130a5fbb77a6b30 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 20 Dec 2024 22:27:29 +0000 Subject: [PATCH 861/962] chore(release): version 20.31.6 [skip ci] ## [20.31.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.5...v20.31.6) (2024-12-20) ### Bug Fixes * Revert changes to disabling auto mode [#3253](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3253) ([#3255](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3255)) ([1ac67b8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/1ac67b8a60e336285c4dca03e550dfc78d64acce)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b62cb191f4..af4a031a26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.31.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.5...v20.31.6) (2024-12-20) + + +### Bug Fixes + +* Revert changes to disabling auto mode [#3253](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3253) ([#3255](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3255)) ([1ac67b8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/1ac67b8a60e336285c4dca03e550dfc78d64acce)) + ## [20.31.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.4...v20.31.5) (2024-12-20) From d876ac4ef1bb45e4f078d0928630033b659c9aa0 Mon Sep 17 00:00:00 2001 From: Anand Capur <551002+arcdigital@users.noreply.github.com> Date: Fri, 17 Jan 2025 08:17:37 -0800 Subject: [PATCH 862/962] feat: Add Bottlerocket FIPS image variants (#3275) fix: Add Bottlerocket FIPS image variants --- modules/_user_data/main.tf | 2 ++ modules/eks-managed-node-group/main.tf | 2 ++ modules/self-managed-node-group/main.tf | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index e66cd3d466..29d5c92d39 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -22,6 +22,8 @@ locals { AL2_ARM_64 = "linux" BOTTLEROCKET_ARM_64 = "bottlerocket" BOTTLEROCKET_x86_64 = "bottlerocket" + BOTTLEROCKET_ARM_64_FIPS = "bottlerocket" + BOTTLEROCKET_x86_64_FIPS = "bottlerocket" BOTTLEROCKET_ARM_64_NVIDIA = "bottlerocket" BOTTLEROCKET_x86_64_NVIDIA = "bottlerocket" WINDOWS_CORE_2019_x86_64 = "windows" diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index c1581439d2..4b15c10b99 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -355,6 +355,8 @@ locals { CUSTOM = "NONE" BOTTLEROCKET_ARM_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/arm64/latest/image_version" BOTTLEROCKET_x86_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/x86_64/latest/image_version" + BOTTLEROCKET_ARM_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/arm64/latest/image_version" + BOTTLEROCKET_x86_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/x86_64/latest/image_version" BOTTLEROCKET_ARM_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/arm64/latest/image_version" BOTTLEROCKET_x86_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/x86_64/latest/image_version" WINDOWS_CORE_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-${local.ssm_cluster_version}" diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 484a92e340..7c651f0e56 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -16,6 +16,8 @@ locals { AL2_ARM_64 = "linux" BOTTLEROCKET_ARM_64 = "bottlerocket" BOTTLEROCKET_x86_64 = "bottlerocket" + BOTTLEROCKET_ARM_64_FIPS = "bottlerocket" + BOTTLEROCKET_x86_64_FIPS = "bottlerocket" BOTTLEROCKET_ARM_64_NVIDIA = "bottlerocket" BOTTLEROCKET_x86_64_NVIDIA = "bottlerocket" WINDOWS_CORE_2019_x86_64 = "windows" @@ -37,6 +39,8 @@ locals { AL2_ARM_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-arm64/recommended/image_id" BOTTLEROCKET_ARM_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/arm64/latest/image_id" BOTTLEROCKET_x86_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/x86_64/latest/image_id" + BOTTLEROCKET_ARM_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/arm64/latest/image_id" + BOTTLEROCKET_x86_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/x86_64/latest/image_id" BOTTLEROCKET_ARM_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/arm64/latest/image_id" BOTTLEROCKET_x86_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/x86_64/latest/image_id" WINDOWS_CORE_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-${local.ssm_cluster_version}/image_id" From 608c41a295a415f9aeea5c397a9dc123cee6d4c9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 17 Jan 2025 16:18:04 +0000 Subject: [PATCH 863/962] chore(release): version 20.32.0 [skip ci] ## [20.32.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.6...v20.32.0) (2025-01-17) ### Features * Add Bottlerocket FIPS image variants ([#3275](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3275)) ([d876ac4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d876ac4ef1bb45e4f078d0928630033b659c9aa0)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index af4a031a26..9daabfaa60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.32.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.6...v20.32.0) (2025-01-17) + + +### Features + +* Add Bottlerocket FIPS image variants ([#3275](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3275)) ([d876ac4](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d876ac4ef1bb45e4f078d0928630033b659c9aa0)) + ## [20.31.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.5...v20.31.6) (2024-12-20) From 4fe03aed68b2cf53005fc5666ee88caa8c4efe19 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 17 Jan 2025 11:00:00 -0600 Subject: [PATCH 864/962] docs: Update v20 upgrade guide to clarify that `"CONFIG_MAP"` is not a supported access type (#3279) --- .pre-commit-config.yaml | 2 +- docs/UPGRADE-20.0.md | 20 ++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index baacea01bb..163a96a55b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.96.2 + rev: v1.97.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/docs/UPGRADE-20.0.md b/docs/UPGRADE-20.0.md index c93a031ee9..629cf395b5 100644 --- a/docs/UPGRADE-20.0.md +++ b/docs/UPGRADE-20.0.md @@ -10,7 +10,7 @@ Please consult the `examples` directory for reference example configurations. If - The default/fallback value for the `preserve` argument of `cluster_addons`is now set to `true`. This has shown to be useful for users deprovisioning clusters while avoiding the situation where the CNI is deleted too early and causes resources to be left orphaned resulting in conflicts. - The Karpenter sub-module's use of the `irsa` naming convention has been removed, along with an update to the Karpenter controller IAM policy to align with Karpenter's `v1beta1`/`v0.32` changes. Instead of referring to the role as `irsa` or `pod_identity`, its simply just an IAM role used by the Karpenter controller and there is support for use with either IRSA and/or Pod Identity (default) at this time - The `aws-auth` ConfigMap resources have been moved to a standalone sub-module. This removes the Kubernetes provider requirement from the main module and allows for the `aws-auth` ConfigMap to be managed independently of the main module. This sub-module will be removed entirely in the next major release. -- Support for cluster access management has been added with the default authentication mode set as `API_AND_CONFIG_MAP`. This is a one way change if applied; if you wish to use `CONFIG_MAP`, you will need to set `authentication_mode = "CONFIG_MAP"` explicitly when upgrading. +- Support for cluster access management has been added with the default authentication mode set as `API_AND_CONFIG_MAP`. Support for `CONFIG_MAP` is no longer supported; instead you will need to use `API_AND_CONFIG_MAP` at minimum - Karpenter EventBridge rule key `spot_interrupt` updated to correct mis-spelling (was `spot_interupt`). This will cause the rule to be replaced ### ⚠️ Upcoming Changes Planned in v21.0 ⚠️ @@ -220,29 +220,13 @@ To give users advanced notice and provide some future direction for this module, Changing the `authentication_mode` is a one-way decision. See [announcement blog](https://aws.amazon.com/blogs/containers/a-deep-dive-into-simplified-amazon-eks-access-management-controls/) for further details: -> Switching authentication modes on an existing cluster is a one-way operation. You can switch from CONFIG_MAP to API_AND_CONFIG_MAP. You can then switch from API_AND_CONFIG_MAP to API. You cannot revert these operations in the opposite direction. Meaning you cannot switch back to CONFIG_MAP or API_AND_CONFIG_MAP from API. And you cannot switch back to CONFIG_MAP from API_AND_CONFIG_MAP. +> Switching authentication modes on an existing cluster is a one-way operation. You can switch from CONFIG_MAP to API_AND_CONFIG_MAP. You can then switch from API_AND_CONFIG_MAP to API. You cannot revert these operations in the opposite direction. Meaning you cannot switch back to CONFIG_MAP or API_AND_CONFIG_MAP from API. > [!IMPORTANT] > If migrating to cluster access entries and you will NOT have any entries that remain in the `aws-auth` ConfigMap, you do not need to remove the configmap from the statefile. You can simply follow the migration guide and once access entries have been created, you can let Terraform remove/delete the `aws-auth` ConfigMap. > > If you WILL have entries that remain in the `aws-auth` ConfigMap, then you will need to remove the ConfigMap resources from the statefile to avoid any disruptions. When you add the new `aws-auth` sub-module and apply the changes, the sub-module will upsert the ConfigMap on the cluster. Provided the necessary entries are defined in that sub-module's definition, it will "re-adopt" the ConfigMap under Terraform's control. -### authentication_mode = "CONFIG_MAP" - -If using `authentication_mode = "CONFIG_MAP"`, before making any changes, you will first need to remove the configmap from the statefile to avoid any disruptions: - -```sh -terraform state rm 'module.eks.kubernetes_config_map_v1_data.aws_auth[0]' -terraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]' # include if Terraform created the original configmap -``` - -Once the configmap has been removed from the statefile, you can add the new `aws-auth` sub-module and copy the relevant definitions from the EKS module over to the new `aws-auth` sub-module definition (see before after diff above). - -> [!CAUTION] -> You will need to add entries to the `aws-auth` sub-module for any IAM roles used by node groups and/or Fargate profiles - the module no longer handles this in the background on behalf of users. -> -> When you apply the changes with the new sub-module, the configmap in the cluster will get updated with the contents provided in the sub-module definition, so please be sure all of the necessary entries are added before applying the changes. - ### authentication_mode = "API_AND_CONFIG_MAP" When using `authentication_mode = "API_AND_CONFIG_MAP"` and there are entries that will remain in the configmap (entries that cannot be replaced by cluster access entry), you will first need to update the `authentication_mode` on the cluster to `"API_AND_CONFIG_MAP"`. To help make this upgrade process easier, a copy of the changes defined in the [`v20.0.0`](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2858) PR have been captured [here](https://github.com/clowdhaus/terraform-aws-eks-v20-migrate) but with the `aws-auth` components still provided in the module. This means you get the equivalent of the `v20.0.0` module, but it still includes support for the `aws-auth` configmap. You can follow the provided README on that interim migration module for the order of execution and return here once the `authentication_mode` has been updated to `"API_AND_CONFIG_MAP"`. Note - EKS automatically adds access entries for the roles used by EKS managed node groups and Fargate profiles; users do not need to do anything additional for these roles. From edd7ef36dd0f6b6801275cbecbb6780f03fc7aed Mon Sep 17 00:00:00 2001 From: Kasper Jacobsen Date: Fri, 17 Jan 2025 18:35:47 +0100 Subject: [PATCH 865/962] feat: Add node repair config to managed node group (#3271) * feat: add var.node_repair_config to eks-managed-node-group Fixes terraform-aws-modules/terraform-aws-eks#3249 * chore: run terraform-docs * chore: update examples/eks-managed-node-group * fix: Correct implementation --------- Co-authored-by: Bryant Biggs --- README.md | 4 ++-- examples/eks-auto-mode/README.md | 4 ++-- examples/eks-auto-mode/versions.tf | 2 +- examples/eks-hybrid-nodes/README.md | 6 +++--- examples/eks-hybrid-nodes/versions.tf | 2 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 6 +++--- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- modules/eks-managed-node-group/README.md | 5 +++-- modules/eks-managed-node-group/main.tf | 8 ++++++++ modules/eks-managed-node-group/variables.tf | 8 ++++++++ modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/hybrid-node-role/README.md | 4 ++-- modules/hybrid-node-role/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- node_groups.tf | 10 +++++----- tests/eks-fargate-profile/README.md | 4 ++-- tests/eks-fargate-profile/versions.tf | 2 +- tests/eks-hybrid-nodes/README.md | 2 +- tests/eks-hybrid-nodes/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/main.tf | 7 +++++++ tests/eks-managed-node-group/versions.tf | 2 +- tests/fast-addons/README.md | 4 ++-- tests/fast-addons/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- versions.tf | 2 +- 34 files changed, 74 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index ca83185353..1d5b5c07be 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -326,7 +326,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index ee2095bf73..b2549472e9 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -25,13 +25,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index 5ca82dce20..9f1a602f58 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -26,7 +26,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | | [helm](#requirement\_helm) | >= 2.16 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | @@ -36,8 +36,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | -| [aws.remote](#provider\_aws.remote) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | +| [aws.remote](#provider\_aws.remote) | >= 5.83 | | [helm](#provider\_helm) | >= 2.16 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index acdd4737dd..124db32301 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } helm = { source = "hashicorp/helm" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 473b1b49c6..f9eef90469 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -94,15 +94,15 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | | [helm](#requirement\_helm) | >= 2.7 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.83 | | [helm](#provider\_helm) | >= 2.7 | ## Modules diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 08067c5339..fa41a8a35f 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 5097c7573f..24e3e9f678 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules @@ -170,6 +170,7 @@ module "eks_managed_node_group" { | [min\_size](#input\_min\_size) | Minimum number of instances/nodes | `number` | `0` | no | | [name](#input\_name) | Name of the EKS managed node group | `string` | `""` | no | | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | +| [node\_repair\_config](#input\_node\_repair\_config) | The node auto repair configuration for the node group |
object({
enabled = optional(bool, true)
})
| `null` | no | | [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | | [placement\_group\_az](#input\_placement\_group\_az) | Availability zone where placement group is created (ex. `eu-west-1c`) | `string` | `null` | no | | [placement\_group\_strategy](#input\_placement\_group\_strategy) | The placement group strategy | `string` | `"cluster"` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 4b15c10b99..19a346613c 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -458,6 +458,14 @@ resource "aws_eks_node_group" "this" { } } + dynamic "node_repair_config" { + for_each = var.node_repair_config != null ? [var.node_repair_config] : [] + + content { + enabled = node_repair_config.value.enabled + } + } + timeouts { create = lookup(var.timeouts, "create", null) update = lookup(var.timeouts, "update", null) diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index b02f42cfcc..54338542c9 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -468,6 +468,14 @@ variable "update_config" { } } +variable "node_repair_config" { + description = "The node auto repair configuration for the node group" + type = object({ + enabled = optional(bool, true) + }) + default = null +} + variable "timeouts" { description = "Create, update, and delete timeout configurations for the node group" type = map(string) diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 1db25fbf57..b636fbe386 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index 3f893571e3..4076fc35f5 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -75,13 +75,13 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 8ef4729463..8baab9d8f6 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index b342044094..9fe09d0998 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/node_groups.tf b/node_groups.tf index d18101253b..8101308441 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -323,11 +323,11 @@ module "eks_managed_node_group" { force_update_version = try(each.value.force_update_version, var.eks_managed_node_group_defaults.force_update_version, null) instance_types = try(each.value.instance_types, var.eks_managed_node_group_defaults.instance_types, null) labels = try(each.value.labels, var.eks_managed_node_group_defaults.labels, null) - - remote_access = try(each.value.remote_access, var.eks_managed_node_group_defaults.remote_access, {}) - taints = try(each.value.taints, var.eks_managed_node_group_defaults.taints, {}) - update_config = try(each.value.update_config, var.eks_managed_node_group_defaults.update_config, local.default_update_config) - timeouts = try(each.value.timeouts, var.eks_managed_node_group_defaults.timeouts, {}) + node_repair_config = try(each.value.node_repair_config, var.eks_managed_node_group_defaults.node_repair_config, null) + remote_access = try(each.value.remote_access, var.eks_managed_node_group_defaults.remote_access, {}) + taints = try(each.value.taints, var.eks_managed_node_group_defaults.taints, {}) + update_config = try(each.value.update_config, var.eks_managed_node_group_defaults.update_config, local.default_update_config) + timeouts = try(each.value.timeouts, var.eks_managed_node_group_defaults.timeouts, {}) # User data platform = try(each.value.platform, var.eks_managed_node_group_defaults.platform, "linux") diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index afe94054f9..7bb85d63f7 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index 6bc9e7d1eb..8f2ac1206c 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -18,7 +18,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index 19bf4edb5e..37eb431fb9 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 1434e52b62..91fc990a92 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index 5d26a53fca..aa73cec30f 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -52,6 +52,9 @@ module "eks" { coredns = { most_recent = true } + eks-node-monitoring-agent = { + most_recent = true + } eks-pod-identity-agent = { before_compute = true most_recent = true @@ -289,6 +292,10 @@ module "eks" { instance_metadata_tags = "disabled" } + node_repair_config = { + enabled = true + } + create_iam_role = true iam_role_name = "eks-managed-node-group-complete-example" iam_role_use_name_prefix = false diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/tests/fast-addons/README.md b/tests/fast-addons/README.md index ae78a4ee73..5c27596125 100644 --- a/tests/fast-addons/README.md +++ b/tests/fast-addons/README.md @@ -22,13 +22,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/tests/fast-addons/versions.tf b/tests/fast-addons/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/tests/fast-addons/versions.tf +++ b/tests/fast-addons/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 3c26d9e435..7a30923161 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.81 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.81 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index e31fee1f34..4736a5b5f7 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } } } diff --git a/versions.tf b/versions.tf index 33405ee6ba..4466790686 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.81" + version = ">= 5.83" } tls = { source = "hashicorp/tls" From 7acf66f8b5ade58689302a86d3857b7d40dc0123 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 17 Jan 2025 17:36:13 +0000 Subject: [PATCH 866/962] chore(release): version 20.33.0 [skip ci] ## [20.33.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.32.0...v20.33.0) (2025-01-17) ### Features * Add node repair config to managed node group ([#3271](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3271)) ([edd7ef3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/edd7ef36dd0f6b6801275cbecbb6780f03fc7aed)), closes [terraform-aws-modules/terraform-aws-eks#3249](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3249) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9daabfaa60..4240e961b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.33.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.32.0...v20.33.0) (2025-01-17) + + +### Features + +* Add node repair config to managed node group ([#3271](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3271)) ([edd7ef3](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/edd7ef36dd0f6b6801275cbecbb6780f03fc7aed)), closes [terraform-aws-modules/terraform-aws-eks#3249](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3249) + ## [20.32.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.31.6...v20.32.0) (2025-01-17) From 3e2ea83267d7532cb66fa4de7f0d2a944b43c3d5 Mon Sep 17 00:00:00 2001 From: Vibham Sharma <45285855+vibhamsharma@users.noreply.github.com> Date: Wed, 22 Jan 2025 10:58:18 +1000 Subject: [PATCH 867/962] fix: Allow `"EC2"` access entry type for EKS Auto Mode custom node pools (#3281) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed EC2 type from condition so policy attachement for access entry can be done for EC2 type for creating access entry for Node role. While Creating Access entry for Self managed node role, we were getting below error as it seems EC2 type is included in exception for policy attachment. Once I removed EC2 from exception condition, it started picking up Policy attachment and Access Entry got created. │ Error: Unsupported attribute │ │ on .terraform/modules/eks/main.tf line 289, in resource "aws_eks_access_policy_association" "this": │ 289: policy_arn = each.value.association_policy_arn │ ├──────────────── │ │ each.value is object with 3 attributes │ │ This object does not have an attribute named "association_policy_arn". --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 3c955d54d0..18933422f5 100644 --- a/main.tf +++ b/main.tf @@ -258,7 +258,7 @@ locals { association_policy_arn = pol_val.policy_arn association_access_scope_type = pol_val.access_scope.type association_access_scope_namespaces = lookup(pol_val.access_scope, "namespaces", []) - } : k => v if !contains(["EC2", "EC2_LINUX", "EC2_WINDOWS", "FARGATE_LINUX", "HYBRID_LINUX"], lookup(entry_val, "type", "STANDARD")) }, + } : k => v if !contains(["EC2_LINUX", "EC2_WINDOWS", "FARGATE_LINUX", "HYBRID_LINUX"], lookup(entry_val, "type", "STANDARD")) }, ) ] ]) From 50cb230c8a1793f5ef4cc52c4c789b656b141168 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 22 Jan 2025 00:58:46 +0000 Subject: [PATCH 868/962] chore(release): version 20.33.1 [skip ci] ## [20.33.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.33.0...v20.33.1) (2025-01-22) ### Bug Fixes * Allow `"EC2"` access entry type for EKS Auto Mode custom node pools ([#3281](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3281)) ([3e2ea83](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3e2ea83267d7532cb66fa4de7f0d2a944b43c3d5)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4240e961b5..d3a2dc3fa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.33.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.33.0...v20.33.1) (2025-01-22) + + +### Bug Fixes + +* Allow `"EC2"` access entry type for EKS Auto Mode custom node pools ([#3281](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3281)) ([3e2ea83](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3e2ea83267d7532cb66fa4de7f0d2a944b43c3d5)) + ## [20.33.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.32.0...v20.33.0) (2025-01-17) From 1bfc10a5589afadf9c633e0417dbf67316e3e32b Mon Sep 17 00:00:00 2001 From: Eddie Bachle Date: Tue, 18 Feb 2025 20:07:34 -0500 Subject: [PATCH 869/962] docs: Update description of karpenter `node_iam_role_use_name_prefix` (#3304) --- modules/karpenter/README.md | 2 +- modules/karpenter/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 8baab9d8f6..fe1956eb80 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -170,7 +170,7 @@ No modules. | [node\_iam\_role\_path](#input\_node\_iam\_role\_path) | IAM role path | `string` | `"/"` | no | | [node\_iam\_role\_permissions\_boundary](#input\_node\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [node\_iam\_role\_tags](#input\_node\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | -| [node\_iam\_role\_use\_name\_prefix](#input\_node\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [node\_iam\_role\_use\_name\_prefix](#input\_node\_iam\_role\_use\_name\_prefix) | Determines whether the Node IAM role name (`node_iam_role_name`) is used as a prefix | `bool` | `true` | no | | [queue\_kms\_data\_key\_reuse\_period\_seconds](#input\_queue\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again | `number` | `null` | no | | [queue\_kms\_master\_key\_id](#input\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no | | [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index a2c307fc05..71b2cbdf38 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -236,7 +236,7 @@ variable "node_iam_role_name" { } variable "node_iam_role_use_name_prefix" { - description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" + description = "Determines whether the Node IAM role name (`node_iam_role_name`) is used as a prefix" type = bool default = true } From 770ee99d9c4b61c509d9988eac62de4db113af91 Mon Sep 17 00:00:00 2001 From: Matt Holmes Date: Fri, 7 Mar 2025 13:31:19 +0000 Subject: [PATCH 870/962] feat: Add capacity reservation permissions to Karpenter IAM policy (#3318) feat: Update AllowScopedEC2InstanceAccessActions policy to include capacity-reservation permission as part of Karpenter version 1.3.0 --- modules/karpenter/policy.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/karpenter/policy.tf b/modules/karpenter/policy.tf index 7fb04e47b2..26c0b235c0 100644 --- a/modules/karpenter/policy.tf +++ b/modules/karpenter/policy.tf @@ -361,6 +361,7 @@ data "aws_iam_policy_document" "v1" { "arn:${local.partition}:ec2:${local.region}::snapshot/*", "arn:${local.partition}:ec2:${local.region}:*:security-group/*", "arn:${local.partition}:ec2:${local.region}:*:subnet/*", + "arn:${local.partition}:ec2:${local.region}:*:capacity-reservation/*", ] actions = [ From dd2089b73b4ff296e519830efdfda760e8d52b8a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 7 Mar 2025 13:31:46 +0000 Subject: [PATCH 871/962] chore(release): version 20.34.0 [skip ci] ## [20.34.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.33.1...v20.34.0) (2025-03-07) ### Features * Add capacity reservation permissions to Karpenter IAM policy ([#3318](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3318)) ([770ee99](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/770ee99d9c4b61c509d9988eac62de4db113af91)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3a2dc3fa1..f4ce42bc45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.34.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.33.1...v20.34.0) (2025-03-07) + + +### Features + +* Add capacity reservation permissions to Karpenter IAM policy ([#3318](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3318)) ([770ee99](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/770ee99d9c4b61c509d9988eac62de4db113af91)) + ## [20.33.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.33.0...v20.33.1) (2025-01-22) From abf76f60144fe645bbf500d98505377fd4a9da79 Mon Sep 17 00:00:00 2001 From: Tom Ellis Date: Sun, 30 Mar 2025 08:51:33 +1100 Subject: [PATCH 872/962] feat: Default to not changing autoscaling schedule values at the scheduled time (#3322) fix: aws_autoscaling_schedule values defaulting to 0 --- modules/eks-managed-node-group/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 19a346613c..8b53ce65c1 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -697,9 +697,9 @@ resource "aws_autoscaling_schedule" "this" { scheduled_action_name = each.key autoscaling_group_name = aws_eks_node_group.this[0].resources[0].autoscaling_groups[0].name - min_size = try(each.value.min_size, null) - max_size = try(each.value.max_size, null) - desired_capacity = try(each.value.desired_size, null) + min_size = try(each.value.min_size, -1) + max_size = try(each.value.max_size, -1) + desired_capacity = try(each.value.desired_size, -1) start_time = try(each.value.start_time, null) end_time = try(each.value.end_time, null) time_zone = try(each.value.time_zone, null) From 0b9d27b3ffb500be824bfdbf2b1607887ef6c295 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 29 Mar 2025 21:52:01 +0000 Subject: [PATCH 873/962] chore(release): version 20.35.0 [skip ci] ## [20.35.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.34.0...v20.35.0) (2025-03-29) ### Features * Default to not changing autoscaling schedule values at the scheduled time ([#3322](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3322)) ([abf76f6](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/abf76f60144fe645bbf500d98505377fd4a9da79)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ce42bc45..99a3ff8854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.35.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.34.0...v20.35.0) (2025-03-29) + + +### Features + +* Default to not changing autoscaling schedule values at the scheduled time ([#3322](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3322)) ([abf76f6](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/abf76f60144fe645bbf500d98505377fd4a9da79)) + ## [20.34.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.33.1...v20.34.0) (2025-03-07) From 5c8ac85c5c428779fd905c2819bb69fd518e7992 Mon Sep 17 00:00:00 2001 From: ismail yenigul Date: Sat, 12 Apr 2025 01:12:41 +0300 Subject: [PATCH 874/962] docs: Set ami_type for AL2023 in the example (#3337) set ami_type for AL2023 Set AMI type explicitly into user data type to be able to use cloudinit_pre_nodeadm https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/modules/_user_data/main.tf#L33 --- examples/eks-managed-node-group/eks-al2023.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/eks-managed-node-group/eks-al2023.tf b/examples/eks-managed-node-group/eks-al2023.tf index 349da821f9..8fe34c62c2 100644 --- a/examples/eks-managed-node-group/eks-al2023.tf +++ b/examples/eks-managed-node-group/eks-al2023.tf @@ -20,6 +20,7 @@ module "eks_al2023" { example = { # Starting on 1.30, AL2023 is the default AMI type for EKS managed node groups instance_types = ["m6i.large"] + ami_type = "AL2023_x86_64_STANDARD" min_size = 2 max_size = 5 From 207d73fbaa5eebe6e98b94e95b83fd0a5a13c307 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 18 Apr 2025 11:29:13 -0500 Subject: [PATCH 875/962] feat: Add support for cluster `force_update_version` (#3345) --- .pre-commit-config.yaml | 2 +- README.md | 5 +++-- examples/eks-auto-mode/README.md | 4 ++-- examples/eks-auto-mode/versions.tf | 2 +- examples/eks-hybrid-nodes/README.md | 6 +++--- examples/eks-hybrid-nodes/versions.tf | 2 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 6 +++--- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- main.tf | 1 + modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/hybrid-node-role/README.md | 4 ++-- modules/hybrid-node-role/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-fargate-profile/README.md | 4 ++-- tests/eks-fargate-profile/versions.tf | 2 +- tests/eks-hybrid-nodes/README.md | 2 +- tests/eks-hybrid-nodes/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/versions.tf | 2 +- tests/fast-addons/README.md | 4 ++-- tests/fast-addons/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- variables.tf | 6 ++++++ versions.tf | 2 +- 33 files changed, 54 insertions(+), 46 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 163a96a55b..34e61cd6c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.97.0 + rev: v1.99.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index 1d5b5c07be..4a511ae8bf 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -326,7 +326,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | @@ -404,6 +404,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `true` | no | | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `false` | no | | [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | +| [cluster\_force\_update\_version](#input\_cluster\_force\_update\_version) | Force version update by overriding upgrade-blocking readiness checks when updating a cluster | `bool` | `null` | no | | [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `"ipv4"` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index b2549472e9..fa7d8fc227 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -25,13 +25,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | ## Modules diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index 9f1a602f58..c092a8f450 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -26,7 +26,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | | [helm](#requirement\_helm) | >= 2.16 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | @@ -36,8 +36,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | -| [aws.remote](#provider\_aws.remote) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | +| [aws.remote](#provider\_aws.remote) | >= 5.95 | | [helm](#provider\_helm) | >= 2.16 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index 124db32301..f392d1edb8 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } helm = { source = "hashicorp/helm" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index f9eef90469..381d32f98e 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -94,15 +94,15 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | | [helm](#requirement\_helm) | >= 2.7 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.95 | | [helm](#provider\_helm) | >= 2.7 | ## Modules diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index fa41a8a35f..6a1b47cbd3 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/main.tf b/main.tf index 18933422f5..c10d79608d 100644 --- a/main.tf +++ b/main.tf @@ -40,6 +40,7 @@ resource "aws_eks_cluster" "this" { version = var.cluster_version enabled_cluster_log_types = var.cluster_enabled_log_types bootstrap_self_managed_addons = local.auto_mode_enabled ? coalesce(var.bootstrap_self_managed_addons, false) : var.bootstrap_self_managed_addons + force_update_version = var.cluster_force_update_version access_config { authentication_mode = var.authentication_mode diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 24e3e9f678..b658c64421 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index b636fbe386..d6ab4ed144 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index 4076fc35f5..50c5a05ec7 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -75,13 +75,13 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | ## Modules diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index fe1956eb80..4523024f79 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 9fe09d0998..fd6f97ad5e 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index 7bb85d63f7..7eeb8a8326 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | ## Modules diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index 8f2ac1206c..5c1a427876 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -18,7 +18,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index 37eb431fb9..afcb64ae1a 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 91fc990a92..4fd29eb844 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/tests/fast-addons/README.md b/tests/fast-addons/README.md index 5c27596125..e2fab2dfa9 100644 --- a/tests/fast-addons/README.md +++ b/tests/fast-addons/README.md @@ -22,13 +22,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | ## Modules diff --git a/tests/fast-addons/versions.tf b/tests/fast-addons/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/tests/fast-addons/versions.tf +++ b/tests/fast-addons/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 7a30923161..437ad94e3d 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.83 | +| [aws](#requirement\_aws) | >= 5.95 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.83 | +| [aws](#provider\_aws) | >= 5.95 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index 4736a5b5f7..c62bcdb38e 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } } } diff --git a/variables.tf b/variables.tf index 855c2133ec..b4881b2988 100644 --- a/variables.tf +++ b/variables.tf @@ -38,6 +38,12 @@ variable "cluster_enabled_log_types" { default = ["audit", "api", "authenticator"] } +variable "cluster_force_update_version" { + description = "Force version update by overriding upgrade-blocking readiness checks when updating a cluster" + type = bool + default = null +} + variable "authentication_mode" { description = "The authentication mode for the cluster. Valid values are `CONFIG_MAP`, `API` or `API_AND_CONFIG_MAP`" type = string diff --git a/versions.tf b/versions.tf index 4466790686..0ce85c882e 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.83" + version = ">= 5.95" } tls = { source = "hashicorp/tls" From 37e3348dffe06ea4b9adf9b54512e4efdb46f425 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 18 Apr 2025 16:29:40 +0000 Subject: [PATCH 876/962] chore(release): version 20.36.0 [skip ci] ## [20.36.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.35.0...v20.36.0) (2025-04-18) ### Features * Add support for cluster `force_update_version` ([#3345](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3345)) ([207d73f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/207d73fbaa5eebe6e98b94e95b83fd0a5a13c307)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99a3ff8854..e0fb947886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.36.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.35.0...v20.36.0) (2025-04-18) + + +### Features + +* Add support for cluster `force_update_version` ([#3345](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3345)) ([207d73f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/207d73fbaa5eebe6e98b94e95b83fd0a5a13c307)) + ## [20.35.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.34.0...v20.35.0) (2025-03-29) From 7a83b1b3db9c7475fe6ec46d1c300c0a18f19b2a Mon Sep 17 00:00:00 2001 From: Pin-Yi <880831ian@gmail.com> Date: Tue, 10 Jun 2025 06:25:06 +0800 Subject: [PATCH 877/962] fix: Ensure `additional_cluster_dns_ips` is passed through from root module (#3376) fix: add missing additional_cluster_dns_ips to node_group module --- node_groups.tf | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/node_groups.tf b/node_groups.tf index 8101308441..de7411f226 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -479,17 +479,18 @@ module "self_managed_node_group" { # User data platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, null) # TODO - update this when `var.platform` is removed in v21.0 - ami_type = try(each.value.ami_type, var.self_managed_node_group_defaults.ami_type, "AL2_x86_64") - cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") - cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") - cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") - cluster_ip_family = var.cluster_ip_family - pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.self_managed_node_group_defaults.pre_bootstrap_user_data, "") - post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "") - bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "") - user_data_template_path = try(each.value.user_data_template_path, var.self_managed_node_group_defaults.user_data_template_path, "") - cloudinit_pre_nodeadm = try(each.value.cloudinit_pre_nodeadm, var.self_managed_node_group_defaults.cloudinit_pre_nodeadm, []) - cloudinit_post_nodeadm = try(each.value.cloudinit_post_nodeadm, var.self_managed_node_group_defaults.cloudinit_post_nodeadm, []) + ami_type = try(each.value.ami_type, var.self_managed_node_group_defaults.ami_type, "AL2_x86_64") + cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") + cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") + cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") + additional_cluster_dns_ips = try(each.value.additional_cluster_dns_ips, var.self_managed_node_group_defaults.additional_cluster_dns_ips, []) + cluster_ip_family = var.cluster_ip_family + pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.self_managed_node_group_defaults.pre_bootstrap_user_data, "") + post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "") + bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "") + user_data_template_path = try(each.value.user_data_template_path, var.self_managed_node_group_defaults.user_data_template_path, "") + cloudinit_pre_nodeadm = try(each.value.cloudinit_pre_nodeadm, var.self_managed_node_group_defaults.cloudinit_pre_nodeadm, []) + cloudinit_post_nodeadm = try(each.value.cloudinit_post_nodeadm, var.self_managed_node_group_defaults.cloudinit_post_nodeadm, []) # Launch Template create_launch_template = try(each.value.create_launch_template, var.self_managed_node_group_defaults.create_launch_template, true) From 89af98d7000ac7e80b61865ba5e5c212ea011938 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 9 Jun 2025 22:25:30 +0000 Subject: [PATCH 878/962] chore(release): version 20.36.1 [skip ci] ## [20.36.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.36.0...v20.36.1) (2025-06-09) ### Bug Fixes * Ensure `additional_cluster_dns_ips` is passed through from root module ([#3376](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3376)) ([7a83b1b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7a83b1b3db9c7475fe6ec46d1c300c0a18f19b2a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0fb947886..26c0994b6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.36.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.36.0...v20.36.1) (2025-06-09) + + +### Bug Fixes + +* Ensure `additional_cluster_dns_ips` is passed through from root module ([#3376](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3376)) ([7a83b1b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/7a83b1b3db9c7475fe6ec46d1c300c0a18f19b2a)) + ## [20.36.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.35.0...v20.36.0) (2025-04-18) From 715d42bf146791cad911b0b6979c5ce67bc0d2f6 Mon Sep 17 00:00:00 2001 From: Nick Baker Date: Mon, 9 Jun 2025 15:27:17 -0700 Subject: [PATCH 879/962] feat: Add AL2023 ARM64 NVIDIA variants (#3369) --- modules/_user_data/main.tf | 1 + modules/eks-managed-node-group/main.tf | 1 + modules/self-managed-node-group/main.tf | 2 ++ 3 files changed, 4 insertions(+) diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index 29d5c92d39..c24074af15 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -34,6 +34,7 @@ locals { AL2023_ARM_64_STANDARD = "al2023" AL2023_x86_64_NEURON = "al2023" AL2023_x86_64_NVIDIA = "al2023" + AL2023_ARM_64_NVIDIA = "al2023" } # Try to use `ami_type` first, but fall back to current, default behavior # TODO - will be removed in v21.0 diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 8b53ce65c1..b9c9ba0e87 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -367,6 +367,7 @@ locals { AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/standard/recommended/release_version" AL2023_x86_64_NEURON = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/neuron/recommended/release_version" AL2023_x86_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/nvidia/recommended/release_version" + AL2023_ARM_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/nvidia/recommended/release_version" } # The Windows SSM params currently do not have a release version, so we have to get the full output JSON blob and parse out the release version diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 7c651f0e56..24454ef74d 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -28,6 +28,7 @@ locals { AL2023_ARM_64_STANDARD = "al2023" AL2023_x86_64_NEURON = "al2023" AL2023_x86_64_NVIDIA = "al2023" + AL2023_ARM_64_NVIDIA = "al2023" } user_data_type = local.ami_type_to_user_data_type[var.ami_type] @@ -51,6 +52,7 @@ locals { AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/standard/recommended/image_id" AL2023_x86_64_NEURON = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/neuron/recommended/image_id" AL2023_x86_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/nvidia/recommended/image_id" + AL2023_ARM_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/nvidia/recommended/image_id" } } From a9c659ad10df816913501e0dd1d9edcc7757dca0 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 9 Jun 2025 22:27:45 +0000 Subject: [PATCH 880/962] chore(release): version 20.37.0 [skip ci] ## [20.37.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.36.1...v20.37.0) (2025-06-09) ### Features * Add AL2023 ARM64 NVIDIA variants ([#3369](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3369)) ([715d42b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/715d42bf146791cad911b0b6979c5ce67bc0d2f6)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c0994b6b..b804b0f1c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.37.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.36.1...v20.37.0) (2025-06-09) + + +### Features + +* Add AL2023 ARM64 NVIDIA variants ([#3369](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3369)) ([715d42b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/715d42bf146791cad911b0b6979c5ce67bc0d2f6)) + ## [20.36.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.36.0...v20.36.1) (2025-06-09) From 681a868d624878474fd9f92d1b04d3fec0120db7 Mon Sep 17 00:00:00 2001 From: Michael Strassberger Date: Wed, 18 Jun 2025 22:08:51 +0200 Subject: [PATCH 881/962] fix: Restrict AWS provider max version due to v6 provider breaking changes (#3384) * Add aws version constraint for 6.0.0 * fix: Restrict all AWS providers and Helm providers upper version limit --------- Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 2 +- README.md | 4 ++-- examples/eks-auto-mode/README.md | 4 ++-- examples/eks-auto-mode/versions.tf | 2 +- examples/eks-hybrid-nodes/README.md | 10 +++++----- examples/eks-hybrid-nodes/versions.tf | 4 ++-- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 10 +++++----- examples/karpenter/versions.tf | 4 ++-- examples/self-managed-node-group/versions.tf | 2 +- modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/hybrid-node-role/README.md | 4 ++-- modules/hybrid-node-role/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-fargate-profile/README.md | 4 ++-- tests/eks-fargate-profile/versions.tf | 2 +- tests/eks-hybrid-nodes/README.md | 2 +- tests/eks-hybrid-nodes/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/versions.tf | 2 +- tests/fast-addons/README.md | 4 ++-- tests/fast-addons/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- versions.tf | 2 +- 31 files changed, 52 insertions(+), 52 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34e61cd6c8..bfddd4b673 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.99.0 + rev: v1.99.4 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index 4a511ae8bf..4b2f4b1b2a 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -326,7 +326,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index fa7d8fc227..085533d59d 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -25,13 +25,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | ## Modules diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index c092a8f450..ed8825f17b 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -26,8 +26,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | -| [helm](#requirement\_helm) | >= 2.16 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [helm](#requirement\_helm) | >= 2.7, < 3.0.0 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | | [tls](#requirement\_tls) | >= 4.0 | @@ -36,9 +36,9 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | -| [aws.remote](#provider\_aws.remote) | >= 5.95 | -| [helm](#provider\_helm) | >= 2.16 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws.remote](#provider\_aws.remote) | >= 5.95, < 6.0.0 | +| [helm](#provider\_helm) | >= 2.7, < 3.0.0 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index f392d1edb8..391c421395 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -4,11 +4,11 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } helm = { source = "hashicorp/helm" - version = ">= 2.16" + version = ">= 2.7, < 3.0.0" } http = { source = "hashicorp/http" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 381d32f98e..7e10796a2d 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -94,16 +94,16 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | -| [helm](#requirement\_helm) | >= 2.7 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [helm](#requirement\_helm) | >= 2.7, < 3.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.95 | -| [helm](#provider\_helm) | >= 2.7 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.95, < 6.0.0 | +| [helm](#provider\_helm) | >= 2.7, < 3.0.0 | ## Modules diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 6a1b47cbd3..f64d6ce245 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,11 +4,11 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } helm = { source = "hashicorp/helm" - version = ">= 2.7" + version = ">= 2.7, < 3.0.0" } } } diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index b658c64421..2eeb0c61c8 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index d6ab4ed144..9b4cb99dd3 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index 50c5a05ec7..3119747f76 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -75,13 +75,13 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | ## Modules diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 4523024f79..43cc3b560b 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index fd6f97ad5e..06747514f2 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index 7eeb8a8326..419f601de8 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | ## Modules diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index 5c1a427876..67bc82c7a9 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -18,7 +18,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index afcb64ae1a..20514e1bdd 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 4fd29eb844..96a5011da3 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/tests/fast-addons/README.md b/tests/fast-addons/README.md index e2fab2dfa9..cd1be0dcd6 100644 --- a/tests/fast-addons/README.md +++ b/tests/fast-addons/README.md @@ -22,13 +22,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | ## Modules diff --git a/tests/fast-addons/versions.tf b/tests/fast-addons/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/tests/fast-addons/versions.tf +++ b/tests/fast-addons/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 437ad94e3d..655025789b 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95 | +| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95 | +| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index c62bcdb38e..7884c4e1b2 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } } } diff --git a/versions.tf b/versions.tf index 0ce85c882e..5024a4995f 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95" + version = ">= 5.95, < 6.0.0" } tls = { source = "hashicorp/tls" From 4c0a8fc4fd534fc039ca075b5bedd56c672d4c5f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 20:09:19 +0000 Subject: [PATCH 882/962] chore(release): version 20.37.1 [skip ci] ## [20.37.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.37.0...v20.37.1) (2025-06-18) ### Bug Fixes * Restrict AWS provider max version due to v6 provider breaking changes ([#3384](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3384)) ([681a868](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/681a868d624878474fd9f92d1b04d3fec0120db7)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b804b0f1c0..85c3f4e506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.37.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.37.0...v20.37.1) (2025-06-18) + + +### Bug Fixes + +* Restrict AWS provider max version due to v6 provider breaking changes ([#3384](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3384)) ([681a868](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/681a868d624878474fd9f92d1b04d3fec0120db7)) + ## [20.37.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.36.1...v20.37.0) (2025-06-09) From 325c3fe29433793573aa5a4c3c455b6967081258 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 1 Jul 2025 16:22:45 -0500 Subject: [PATCH 883/962] docs: Update doc references to `1.33`; remove AL2 examples since `1.33`+ does not support AL2 (#3402) --- README.md | 6 ++-- docs/compute_resources.md | 4 +-- examples/eks-auto-mode/main.tf | 2 +- .../eks-hybrid-nodes/ami/variables.pkr.hcl | 2 +- examples/eks-hybrid-nodes/main.tf | 2 +- examples/eks-managed-node-group/README.md | 1 - examples/eks-managed-node-group/eks-al2.tf | 34 ------------------- examples/eks-managed-node-group/eks-al2023.tf | 2 +- .../eks-bottlerocket.tf | 2 +- examples/karpenter/main.tf | 2 +- examples/self-managed-node-group/README.md | 1 - examples/self-managed-node-group/eks-al2.tf | 33 ------------------ .../self-managed-node-group/eks-al2023.tf | 2 +- .../eks-bottlerocket.tf | 2 +- tests/eks-fargate-profile/main.tf | 2 +- tests/eks-managed-node-group/main.tf | 2 +- tests/fast-addons/main.tf | 2 +- tests/self-managed-node-group/main.tf | 2 +- 18 files changed, 17 insertions(+), 86 deletions(-) delete mode 100644 examples/eks-managed-node-group/eks-al2.tf delete mode 100644 examples/self-managed-node-group/eks-al2.tf diff --git a/README.md b/README.md index 4b2f4b1b2a..4767a624e1 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ module "eks" { version = "~> 20.31" cluster_name = "example" - cluster_version = "1.31" + cluster_version = "1.33" # Optional cluster_endpoint_public_access = true @@ -82,7 +82,7 @@ module "eks" { version = "~> 20.31" cluster_name = "example" - cluster_version = "1.31" + cluster_version = "1.33" cluster_addons = { coredns = {} @@ -149,7 +149,7 @@ module "eks" { version = "~> 20.0" cluster_name = "my-cluster" - cluster_version = "1.31" + cluster_version = "1.33" bootstrap_self_managed_addons = false cluster_addons = { diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 6b74c4a507..a7bd5b5e93 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -113,7 +113,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 1. The `self-managed-node-group` uses the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version by default: ```hcl - cluster_version = "1.31" + cluster_version = "1.33" # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.27 self_managed_node_groups = { @@ -124,7 +124,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 2. To use Bottlerocket, specify the `ami_type` as one of the respective `"BOTTLEROCKET_*" types` and supply a Bottlerocket OS AMI: ```hcl - cluster_version = "1.31" + cluster_version = "1.33" self_managed_node_groups = { bottlerocket = { diff --git a/examples/eks-auto-mode/main.tf b/examples/eks-auto-mode/main.tf index 756437d297..316b2764d4 100644 --- a/examples/eks-auto-mode/main.tf +++ b/examples/eks-auto-mode/main.tf @@ -12,7 +12,7 @@ data "aws_availability_zones" "available" { locals { name = "ex-${basename(path.cwd)}" - cluster_version = "1.31" + cluster_version = "1.33" region = "us-west-2" vpc_cidr = "10.0.0.0/16" diff --git a/examples/eks-hybrid-nodes/ami/variables.pkr.hcl b/examples/eks-hybrid-nodes/ami/variables.pkr.hcl index 89e1b27ffa..1eb9c4726b 100644 --- a/examples/eks-hybrid-nodes/ami/variables.pkr.hcl +++ b/examples/eks-hybrid-nodes/ami/variables.pkr.hcl @@ -7,7 +7,7 @@ variable "ami_name_prefix" { variable "eks_version" { description = "The EKS cluster version associated with the AMI created" type = string - default = "1.31" + default = "1.33" } variable "credential_provider" { diff --git a/examples/eks-hybrid-nodes/main.tf b/examples/eks-hybrid-nodes/main.tf index 8791e6cb53..ffc77e36a2 100644 --- a/examples/eks-hybrid-nodes/main.tf +++ b/examples/eks-hybrid-nodes/main.tf @@ -20,7 +20,7 @@ locals { name = "ex-${basename(path.cwd)}" region = "us-west-2" - cluster_version = "1.31" + cluster_version = "1.33" tags = { Test = local.name diff --git a/examples/eks-managed-node-group/README.md b/examples/eks-managed-node-group/README.md index a75636480a..16708dca42 100644 --- a/examples/eks-managed-node-group/README.md +++ b/examples/eks-managed-node-group/README.md @@ -2,7 +2,6 @@ Configuration in this directory creates Amazon EKS clusters with EKS Managed Node Groups demonstrating different configurations: -- `eks-al2.tf` demonstrates an EKS cluster using EKS managed node group that utilizes the EKS Amazon Linux 2 optimized AMI - `eks-al2023.tf` demonstrates an EKS cluster using EKS managed node group that utilizes the EKS Amazon Linux 2023 optimized AMI - `eks-bottlerocket.tf` demonstrates an EKS cluster using EKS managed node group that utilizes the Bottlerocket EKS optimized AMI diff --git a/examples/eks-managed-node-group/eks-al2.tf b/examples/eks-managed-node-group/eks-al2.tf deleted file mode 100644 index 910fa216b2..0000000000 --- a/examples/eks-managed-node-group/eks-al2.tf +++ /dev/null @@ -1,34 +0,0 @@ -module "eks_al2" { - source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" - - cluster_name = "${local.name}-al2" - cluster_version = "1.31" - - # EKS Addons - cluster_addons = { - coredns = {} - eks-pod-identity-agent = {} - kube-proxy = {} - vpc-cni = {} - } - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - eks_managed_node_groups = { - example = { - # Starting on 1.30, AL2023 is the default AMI type for EKS managed node groups - ami_type = "AL2_x86_64" - instance_types = ["m6i.large"] - - min_size = 2 - max_size = 5 - # This value is ignored after the initial creation - # https://github.com/bryantbiggs/eks-desired-size-hack - desired_size = 2 - } - } - - tags = local.tags -} diff --git a/examples/eks-managed-node-group/eks-al2023.tf b/examples/eks-managed-node-group/eks-al2023.tf index 8fe34c62c2..93fda067da 100644 --- a/examples/eks-managed-node-group/eks-al2023.tf +++ b/examples/eks-managed-node-group/eks-al2023.tf @@ -3,7 +3,7 @@ module "eks_al2023" { version = "~> 20.0" cluster_name = "${local.name}-al2023" - cluster_version = "1.31" + cluster_version = "1.33" # EKS Addons cluster_addons = { diff --git a/examples/eks-managed-node-group/eks-bottlerocket.tf b/examples/eks-managed-node-group/eks-bottlerocket.tf index 01a6878814..4d25e5b9c0 100644 --- a/examples/eks-managed-node-group/eks-bottlerocket.tf +++ b/examples/eks-managed-node-group/eks-bottlerocket.tf @@ -3,7 +3,7 @@ module "eks_bottlerocket" { version = "~> 20.0" cluster_name = "${local.name}-bottlerocket" - cluster_version = "1.31" + cluster_version = "1.33" # EKS Addons cluster_addons = { diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 9961063c95..7006740f47 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -55,7 +55,7 @@ module "eks" { source = "../.." cluster_name = local.name - cluster_version = "1.31" + cluster_version = "1.33" # Gives Terraform identity admin access to cluster which will # allow deploying resources (Karpenter) into the cluster diff --git a/examples/self-managed-node-group/README.md b/examples/self-managed-node-group/README.md index da81c84142..ce84716f8a 100644 --- a/examples/self-managed-node-group/README.md +++ b/examples/self-managed-node-group/README.md @@ -2,7 +2,6 @@ Configuration in this directory creates Amazon EKS clusters with self-managed node groups demonstrating different configurations: -- `eks-al2.tf` demonstrates an EKS cluster using self-managed node group that utilizes the EKS Amazon Linux 2 optimized AMI - `eks-al2023.tf` demonstrates an EKS cluster using self-managed node group that utilizes the EKS Amazon Linux 2023 optimized AMI - `eks-bottlerocket.tf` demonstrates an EKS cluster using self-managed node group that utilizes the Bottlerocket EKS optimized AMI diff --git a/examples/self-managed-node-group/eks-al2.tf b/examples/self-managed-node-group/eks-al2.tf deleted file mode 100644 index e015982733..0000000000 --- a/examples/self-managed-node-group/eks-al2.tf +++ /dev/null @@ -1,33 +0,0 @@ -module "eks_al2" { - source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" - - cluster_name = "${local.name}-al2" - cluster_version = "1.31" - - # EKS Addons - cluster_addons = { - coredns = {} - eks-pod-identity-agent = {} - kube-proxy = {} - vpc-cni = {} - } - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - self_managed_node_groups = { - example = { - ami_type = "AL2_x86_64" - instance_type = "m6i.large" - - min_size = 2 - max_size = 5 - # This value is ignored after the initial creation - # https://github.com/bryantbiggs/eks-desired-size-hack - desired_size = 2 - } - } - - tags = local.tags -} diff --git a/examples/self-managed-node-group/eks-al2023.tf b/examples/self-managed-node-group/eks-al2023.tf index 4d7c5dc7e1..58ecb3cf66 100644 --- a/examples/self-managed-node-group/eks-al2023.tf +++ b/examples/self-managed-node-group/eks-al2023.tf @@ -3,7 +3,7 @@ module "eks_al2023" { version = "~> 20.0" cluster_name = "${local.name}-al2023" - cluster_version = "1.31" + cluster_version = "1.33" # EKS Addons cluster_addons = { diff --git a/examples/self-managed-node-group/eks-bottlerocket.tf b/examples/self-managed-node-group/eks-bottlerocket.tf index ff2fb2cf3d..adb3462b82 100644 --- a/examples/self-managed-node-group/eks-bottlerocket.tf +++ b/examples/self-managed-node-group/eks-bottlerocket.tf @@ -3,7 +3,7 @@ module "eks_bottlerocket" { version = "~> 20.0" cluster_name = "${local.name}-bottlerocket" - cluster_version = "1.31" + cluster_version = "1.33" # EKS Addons cluster_addons = { diff --git a/tests/eks-fargate-profile/main.tf b/tests/eks-fargate-profile/main.tf index 37fc44b3f9..bf91d522a8 100644 --- a/tests/eks-fargate-profile/main.tf +++ b/tests/eks-fargate-profile/main.tf @@ -12,7 +12,7 @@ data "aws_availability_zones" "available" { locals { name = "ex-${basename(path.cwd)}" - cluster_version = "1.31" + cluster_version = "1.33" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index aa73cec30f..010a55fa1b 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -14,7 +14,7 @@ data "aws_availability_zones" "available" { locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.31" + cluster_version = "1.33" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" diff --git a/tests/fast-addons/main.tf b/tests/fast-addons/main.tf index 12d2a41f8c..e7a5b9cc55 100644 --- a/tests/fast-addons/main.tf +++ b/tests/fast-addons/main.tf @@ -4,7 +4,7 @@ provider "aws" { locals { name = "ex-${basename(path.cwd)}" - cluster_version = "1.31" + cluster_version = "1.33" region = "eu-west-1" tags = { diff --git a/tests/self-managed-node-group/main.tf b/tests/self-managed-node-group/main.tf index f2181fdc75..351985f359 100644 --- a/tests/self-managed-node-group/main.tf +++ b/tests/self-managed-node-group/main.tf @@ -14,7 +14,7 @@ data "aws_availability_zones" "available" { locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.31" + cluster_version = "1.33" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" From 83b68fda2b0ea818fc980ab847dd8255a2d18334 Mon Sep 17 00:00:00 2001 From: datty Date: Thu, 17 Jul 2025 15:03:09 +0100 Subject: [PATCH 884/962] fix: Allow for both `amazonaws.com.cn` and `amazonaws.com` conditions in PassRole as required for AWS CN (#3422) * Allow for both amazonaws.com.cn and amazonaws.com conditions as required for AWS CN * Allow for both amazonaws.com.cn and amazonaws.com conditions as required for AWS CN - set in correct policy --------- Co-authored-by: Oliver Smith --- modules/karpenter/policy.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/karpenter/policy.tf b/modules/karpenter/policy.tf index 26c0b235c0..b9068d7b60 100644 --- a/modules/karpenter/policy.tf +++ b/modules/karpenter/policy.tf @@ -585,7 +585,7 @@ data "aws_iam_policy_document" "v1" { condition { test = "StringEquals" variable = "iam:PassedToService" - values = ["ec2.${local.dns_suffix}"] + values = distinct(["ec2.${local.dns_suffix}", "ec2.amazonaws.com"]) } } From 8a0efdbbc84180a26e0bacfd2b6fcfceac53b3b6 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 17 Jul 2025 14:03:44 +0000 Subject: [PATCH 885/962] chore(release): version 20.37.2 [skip ci] ## [20.37.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.37.1...v20.37.2) (2025-07-17) ### Bug Fixes * Allow for both `amazonaws.com.cn` and `amazonaws.com` conditions in PassRole as required for AWS CN ([#3422](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3422)) ([83b68fd](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/83b68fda2b0ea818fc980ab847dd8255a2d18334)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85c3f4e506..cb8e0eee24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [20.37.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.37.1...v20.37.2) (2025-07-17) + + +### Bug Fixes + +* Allow for both `amazonaws.com.cn` and `amazonaws.com` conditions in PassRole as required for AWS CN ([#3422](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3422)) ([83b68fd](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/83b68fda2b0ea818fc980ab847dd8255a2d18334)) + ## [20.37.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.37.0...v20.37.1) (2025-06-18) From 416515a0da1ca96c539977d6460e2bc02f10b4d4 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 23 Jul 2025 15:11:01 -0500 Subject: [PATCH 886/962] feat!: Upgrade min AWS provider and Terraform versions to `6.0` and `1.5.7` respectively (#3412) * feat!: Upgrade min AWS provider and Terraform versions to `6.0` and `1.5.7` respectively * fix: Remove deprecated arguments in AWS v6.0 provider, upgrade Helm provider to v3.0, bump VPC module to v6.0 * fix: Remove `aws-auth` sub-module * fix: Remove `platform` and `cluster_service_ipv4_cidr` variables from `user-data` sub-module * fix: Resolve all marked `todos` that have been accumulated * fix: Set default `http_put_response_hop_limit` to `1` * fix: Remove IRSA support from Karpenter sub-module * fix: Avoid making GET requests from data sources unless absolutely necessary * feat: Add variable optional attribute definitions * feat: Bump KMS key module version to latest, add remaining variable attribute definitions * fix: Remove `cluster_` prefix from variable names to better match the underlying API * fix: Move all EFA logic to the nodegroup itself * fix: Remove arguments that do not make sense in EKS * fix: Updates from plan validation * fix: Remove more self-managed node group attributes that are commonly not used in EKS clusters * fix: Remove data plane compute `*_defaults` variables that do not work with variable optional attributes * fix: Ignore changes to `bootstrap_self_managed_addons` to aid in upgrade * feat: Add support for `region` argument on relevant resources * feat: Initial pass on upgrade guide * fix: Updates from testing and validating EKS managed node group * fix: Updates from testing and validating self-managed node group * docs: Ensure addon ussage documented is aligned * feat: Switch to dualstack OIDC issuer URL * feat: Allow sourcing over overriding the Karpenter assume role policy * fix: Use `Bool` instead of `StringEquals` for DenyHTTP queue policy * fix: Correct use of `nullable` and default value propagation --- .pre-commit-config.yaml | 2 +- README.md | 296 +++-- docs/README.md | 1 + docs/UPGRADE-21.0.md | 328 ++++++ docs/compute_resources.md | 77 +- docs/faq.md | 43 +- docs/network_connectivity.md | 2 +- docs/user_data.md | 24 +- examples/eks-auto-mode/README.md | 8 +- examples/eks-auto-mode/main.tf | 16 +- examples/eks-auto-mode/versions.tf | 4 +- examples/eks-hybrid-nodes/README.md | 16 +- examples/eks-hybrid-nodes/main.tf | 22 +- examples/eks-hybrid-nodes/remote.tf | 4 +- examples/eks-hybrid-nodes/versions.tf | 6 +- examples/eks-managed-node-group/eks-al2023.tf | 25 +- .../eks-bottlerocket.tf | 23 +- examples/eks-managed-node-group/main.tf | 2 +- examples/eks-managed-node-group/versions.tf | 4 +- examples/karpenter/README.md | 14 +- examples/karpenter/main.tf | 31 +- examples/karpenter/versions.tf | 6 +- .../self-managed-node-group/eks-al2023.tf | 25 +- .../eks-bottlerocket.tf | 23 +- examples/self-managed-node-group/main.tf | 2 +- examples/self-managed-node-group/versions.tf | 4 +- main.tf | 329 +++--- modules/_user_data/README.md | 9 +- modules/_user_data/main.tf | 97 +- modules/_user_data/outputs.tf | 7 +- modules/_user_data/variables.tf | 35 +- modules/_user_data/versions.tf | 2 +- modules/aws-auth/README.md | 81 -- modules/aws-auth/main.tf | 47 - modules/aws-auth/outputs.tf | 0 modules/aws-auth/variables.tf | 39 - modules/aws-auth/versions.tf | 10 - modules/eks-managed-node-group/README.md | 104 +- modules/eks-managed-node-group/main.tf | 469 ++++---- modules/eks-managed-node-group/outputs.tf | 22 +- modules/eks-managed-node-group/variables.tf | 382 ++++-- modules/eks-managed-node-group/versions.tf | 4 +- modules/fargate-profile/README.md | 19 +- modules/fargate-profile/main.tf | 61 +- modules/fargate-profile/variables.tf | 71 +- modules/fargate-profile/versions.tf | 4 +- modules/hybrid-node-role/README.md | 14 +- modules/hybrid-node-role/main.tf | 44 +- modules/hybrid-node-role/variables.tf | 59 +- modules/hybrid-node-role/versions.tf | 4 +- modules/karpenter/README.md | 23 +- modules/karpenter/main.tf | 100 +- modules/karpenter/policy.tf | 375 +----- modules/karpenter/variables.tf | 89 +- modules/karpenter/versions.tf | 4 +- modules/self-managed-node-group/README.md | 124 +- modules/self-managed-node-group/main.tf | 843 +++++++------- modules/self-managed-node-group/outputs.tf | 28 +- modules/self-managed-node-group/variables.tf | 596 +++++++--- modules/self-managed-node-group/versions.tf | 4 +- node_groups.tf | 607 +++++----- outputs.tf | 3 +- ...{linux_user_data.tpl => al2_user_data.tpl} | 0 tests/eks-fargate-profile/README.md | 8 +- tests/eks-fargate-profile/main.tf | 30 +- tests/eks-fargate-profile/versions.tf | 4 +- tests/eks-hybrid-nodes/README.md | 4 +- tests/eks-hybrid-nodes/versions.tf | 4 +- tests/eks-managed-node-group/README.md | 10 +- tests/eks-managed-node-group/main.tf | 49 +- tests/eks-managed-node-group/versions.tf | 4 +- tests/fast-addons/README.md | 92 -- tests/fast-addons/main.tf | 159 --- tests/fast-addons/outputs.tf | 226 ---- tests/fast-addons/variables.tf | 0 tests/fast-addons/versions.tf | 10 - tests/self-managed-node-group/README.md | 12 +- tests/self-managed-node-group/main.tf | 114 +- tests/self-managed-node-group/versions.tf | 4 +- tests/user-data/README.md | 2 +- tests/user-data/main.tf | 15 +- tests/user-data/versions.tf | 2 +- variables.tf | 1028 +++++++++++++++-- versions.tf | 6 +- 84 files changed, 4136 insertions(+), 3364 deletions(-) create mode 100644 docs/UPGRADE-21.0.md delete mode 100644 modules/aws-auth/README.md delete mode 100644 modules/aws-auth/main.tf delete mode 100644 modules/aws-auth/outputs.tf delete mode 100644 modules/aws-auth/variables.tf delete mode 100644 modules/aws-auth/versions.tf rename templates/{linux_user_data.tpl => al2_user_data.tpl} (100%) delete mode 100644 tests/fast-addons/README.md delete mode 100644 tests/fast-addons/main.tf delete mode 100644 tests/fast-addons/outputs.tf delete mode 100644 tests/fast-addons/variables.tf delete mode 100644 tests/fast-addons/versions.tf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bfddd4b673..b489ccee0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.99.4 + rev: v1.99.5 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index 4767a624e1..897d445231 100644 --- a/README.md +++ b/README.md @@ -30,18 +30,18 @@ Please note that we strive to provide a comprehensive suite of documentation for ```hcl module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 20.31" + version = "~> 21.0" - cluster_name = "example" - cluster_version = "1.33" + name = "example" + kubernetes_version = "1.33" # Optional - cluster_endpoint_public_access = true + endpoint_public_access = true # Optional: Adds the current caller identity as an administrator via cluster access entry enable_cluster_creator_admin_permissions = true - cluster_compute_config = { + compute_config = { enabled = true node_pools = ["general-purpose"] } @@ -56,6 +56,89 @@ module "eks" { } ``` +### EKS Managed Node Group + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" + + name = "my-cluster" + kubernetes_version = "1.33" + + addons = { + coredns = {} + eks-pod-identity-agent = { + before_compute = true + } + kube-proxy = {} + vpc-cni = { + before_compute = true + } + } + + # Optional + endpoint_public_access = true + + # Optional: Adds the current caller identity as an administrator via cluster access entry + enable_cluster_creator_admin_permissions = true + + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + control_plane_subnet_ids = ["subnet-xyzde987", "subnet-slkjf456", "subnet-qeiru789"] + + # EKS Managed Node Group(s) + eks_managed_node_groups = { + example = { + # Starting on 1.30, AL2023 is the default AMI type for EKS managed node groups + ami_type = "AL2023_x86_64_STANDARD" + instance_types = ["m5.xlarge"] + + min_size = 2 + max_size = 10 + desired_size = 2 + } + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### Cluster Access Entry + +When enabling `authentication_mode = "API_AND_CONFIG_MAP"`, EKS will automatically create an access entry for the IAM role(s) used by managed node group(s) and Fargate profile(s). There are no additional actions required by users. For self-managed node groups and the Karpenter sub-module, this project automatically adds the access entry on behalf of users so there are no additional actions required by users. + +On clusters that were created prior to cluster access management (CAM) support, there will be an existing access entry for the cluster creator. This was previously not visible when using `aws-auth` ConfigMap, but will become visible when access entry is enabled. + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" + + # Truncated for brevity ... + + access_entries = { + # One access entry with a policy associated + example = { + principal_arn = "arn:aws:iam::123456789012:role/something" + + policy_associations = { + example = { + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy" + access_scope = { + namespaces = ["default"] + type = "namespace" + } + } + } + } + } +} +``` + ### EKS Hybrid Nodes ```hcl @@ -69,7 +152,7 @@ locals { # SSM and IAM Roles Anywhere supported - SSM is default module "eks_hybrid_node_role" { source = "terraform-aws-modules/eks/aws//modules/hybrid-node-role" - version = "~> 20.31" + version = "~> 21.0" tags = { Environment = "dev" @@ -79,25 +162,25 @@ module "eks_hybrid_node_role" { module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 20.31" + version = "~> 21.0" - cluster_name = "example" - cluster_version = "1.33" + name = "example" + kubernetes_version = "1.33" - cluster_addons = { + addons = { coredns = {} eks-pod-identity-agent = {} kube-proxy = {} } # Optional - cluster_endpoint_public_access = true + endpoint_public_access = true # Optional: Adds the current caller identity as an administrator via cluster access entry enable_cluster_creator_admin_permissions = true create_node_security_group = false - cluster_security_group_additional_rules = { + security_group_additional_rules = { hybrid-all = { cidr_blocks = [local.remote_network_cidr] description = "Allow all traffic from remote node/pod network" @@ -109,7 +192,7 @@ module "eks" { } # Optional - cluster_compute_config = { + compute_config = { enabled = true node_pools = ["system"] } @@ -124,7 +207,7 @@ module "eks" { vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] - cluster_remote_network_config = { + remote_network_config = { remote_node_networks = { cidrs = [local.remote_node_cidr] } @@ -141,90 +224,6 @@ module "eks" { } ``` -### EKS Managed Node Group - -```hcl -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" - - cluster_name = "my-cluster" - cluster_version = "1.33" - - bootstrap_self_managed_addons = false - cluster_addons = { - coredns = {} - eks-pod-identity-agent = {} - kube-proxy = {} - vpc-cni = {} - } - - # Optional - cluster_endpoint_public_access = true - - # Optional: Adds the current caller identity as an administrator via cluster access entry - enable_cluster_creator_admin_permissions = true - - vpc_id = "vpc-1234556abcdef" - subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] - control_plane_subnet_ids = ["subnet-xyzde987", "subnet-slkjf456", "subnet-qeiru789"] - - # EKS Managed Node Group(s) - eks_managed_node_group_defaults = { - instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] - } - - eks_managed_node_groups = { - example = { - # Starting on 1.30, AL2023 is the default AMI type for EKS managed node groups - ami_type = "AL2023_x86_64_STANDARD" - instance_types = ["m5.xlarge"] - - min_size = 2 - max_size = 10 - desired_size = 2 - } - } - - tags = { - Environment = "dev" - Terraform = "true" - } -} -``` - -### Cluster Access Entry - -When enabling `authentication_mode = "API_AND_CONFIG_MAP"`, EKS will automatically create an access entry for the IAM role(s) used by managed node group(s) and Fargate profile(s). There are no additional actions required by users. For self-managed node groups and the Karpenter sub-module, this project automatically adds the access entry on behalf of users so there are no additional actions required by users. - -On clusters that were created prior to cluster access management (CAM) support, there will be an existing access entry for the cluster creator. This was previously not visible when using `aws-auth` ConfigMap, but will become visible when access entry is enabled. - -```hcl -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" - - # Truncated for brevity ... - - access_entries = { - # One access entry with a policy associated - example = { - principal_arn = "arn:aws:iam::123456789012:role/something" - - policy_associations = { - example = { - policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy" - access_scope = { - namespaces = ["default"] - type = "namespace" - } - } - } - } - } -} -``` - ### Bootstrap Cluster Creator Admin Permissions Setting the `bootstrap_cluster_creator_admin_permissions` is a one time operation when the cluster is created; it cannot be modified later through the EKS API. In this project we are hardcoding this to `false`. If users wish to achieve the same functionality, we will do that through an access entry which can be enabled or disabled at any time of their choosing using the variable `enable_cluster_creator_admin_permissions` @@ -251,7 +250,7 @@ In order to enable EFA support, you will have to specify `enable_efa_support = t ```hcl module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" + version = "~> 21.0" # Truncated for brevity ... @@ -317,18 +316,18 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | | [time](#requirement\_time) | >= 0.9 | -| [tls](#requirement\_tls) | >= 3.0 | +| [tls](#requirement\_tls) | >= 4.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws](#provider\_aws) | >= 6.0 | | [time](#provider\_time) | >= 0.9 | -| [tls](#provider\_tls) | >= 3.0 | +| [tls](#provider\_tls) | >= 4.0 | ## Modules @@ -336,7 +335,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|--------|---------| | [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ./modules/eks-managed-node-group | n/a | | [fargate\_profile](#module\_fargate\_profile) | ./modules/fargate-profile | n/a | -| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 2.1.0 | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | 4.0.0 | | [self\_managed\_node\_group](#module\_self\_managed\_node\_group) | ./modules/self-managed-node-group | n/a | ## Resources @@ -382,68 +381,47 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [access\_entries](#input\_access\_entries) | Map of access entries to add to the cluster | `any` | `{}` | no | -| [attach\_cluster\_encryption\_policy](#input\_attach\_cluster\_encryption\_policy) | Indicates whether or not to attach an additional policy for the cluster IAM role to utilize the encryption key provided | `bool` | `true` | no | +| [access\_entries](#input\_access\_entries) | Map of access entries to add to the cluster |
map(object({
# Access entry
kubernetes_groups = optional(list(string))
principal_arn = string
type = optional(string, "STANDARD")
user_name = optional(string)
tags = optional(map(string), {})
# Access policy association
policy_associations = optional(map(object({
policy_arn = string
access_scope = object({
namespaces = optional(list(string))
type = string
})
})))
}))
| `{}` | no | +| [additional\_security\_group\_ids](#input\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | +| [addons](#input\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` |
map(object({
name = optional(string) # will fall back to map key
before_compute = optional(bool, false)
most_recent = optional(bool, true)
addon_version = optional(string)
configuration_values = optional(string)
pod_identity_association = optional(list(object({
role_arn = string
service_account = string
})))
preserve = optional(bool, true)
resolve_conflicts_on_create = optional(string, "NONE")
resolve_conflicts_on_update = optional(string, "OVERWRITE")
service_account_role_arn = optional(string)
timeouts = optional(object({
create = optional(string)
update = optional(string)
delete = optional(string)
}))
tags = optional(map(string), {})
}))
| `null` | no | +| [addons\_timeouts](#input\_addons\_timeouts) | Create, update, and delete timeout configurations for the cluster addons |
object({
create = optional(string)
update = optional(string)
delete = optional(string)
})
| `null` | no | +| [attach\_encryption\_policy](#input\_attach\_encryption\_policy) | Indicates whether or not to attach an additional policy for the cluster IAM role to utilize the encryption key provided | `bool` | `true` | no | | [authentication\_mode](#input\_authentication\_mode) | The authentication mode for the cluster. Valid values are `CONFIG_MAP`, `API` or `API_AND_CONFIG_MAP` | `string` | `"API_AND_CONFIG_MAP"` | no | -| [bootstrap\_self\_managed\_addons](#input\_bootstrap\_self\_managed\_addons) | Indicates whether or not to bootstrap self-managed addons after the cluster has been created | `bool` | `null` | no | | [cloudwatch\_log\_group\_class](#input\_cloudwatch\_log\_group\_class) | Specified the log class of the log group. Possible values are: `STANDARD` or `INFREQUENT_ACCESS` | `string` | `null` | no | | [cloudwatch\_log\_group\_kms\_key\_id](#input\_cloudwatch\_log\_group\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `null` | no | | [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days | `number` | `90` | no | | [cloudwatch\_log\_group\_tags](#input\_cloudwatch\_log\_group\_tags) | A map of additional tags to add to the cloudwatch log group created | `map(string)` | `{}` | no | -| [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | -| [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no | -| [cluster\_addons\_timeouts](#input\_cluster\_addons\_timeouts) | Create, update, and delete timeout configurations for the cluster addons | `map(string)` | `{}` | no | -| [cluster\_compute\_config](#input\_cluster\_compute\_config) | Configuration block for the cluster compute configuration | `any` | `{}` | no | -| [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | -| [cluster\_encryption\_config](#input\_cluster\_encryption\_config) | Configuration block with encryption configuration for the cluster. To disable secret encryption, set this value to `{}` | `any` |
{
"resources": [
"secrets"
]
}
| no | -| [cluster\_encryption\_policy\_description](#input\_cluster\_encryption\_policy\_description) | Description of the cluster encryption policy created | `string` | `"Cluster encryption policy to allow cluster role to utilize CMK provided"` | no | -| [cluster\_encryption\_policy\_name](#input\_cluster\_encryption\_policy\_name) | Name to use on cluster encryption policy created | `string` | `null` | no | -| [cluster\_encryption\_policy\_path](#input\_cluster\_encryption\_policy\_path) | Cluster encryption policy path | `string` | `null` | no | -| [cluster\_encryption\_policy\_tags](#input\_cluster\_encryption\_policy\_tags) | A map of additional tags to add to the cluster encryption policy created | `map(string)` | `{}` | no | -| [cluster\_encryption\_policy\_use\_name\_prefix](#input\_cluster\_encryption\_policy\_use\_name\_prefix) | Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix | `bool` | `true` | no | -| [cluster\_endpoint\_private\_access](#input\_cluster\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `true` | no | -| [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `false` | no | -| [cluster\_endpoint\_public\_access\_cidrs](#input\_cluster\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | -| [cluster\_force\_update\_version](#input\_cluster\_force\_update\_version) | Force version update by overriding upgrade-blocking readiness checks when updating a cluster | `bool` | `null` | no | -| [cluster\_identity\_providers](#input\_cluster\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA | `any` | `{}` | no | -| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `"ipv4"` | no | -| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | -| [cluster\_remote\_network\_config](#input\_cluster\_remote\_network\_config) | Configuration block for the cluster remote network configuration | `any` | `{}` | no | -| [cluster\_security\_group\_additional\_rules](#input\_cluster\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created. Set `source_node_security_group = true` inside rules to set the `node_security_group` as source | `any` | `{}` | no | -| [cluster\_security\_group\_description](#input\_cluster\_security\_group\_description) | Description of the cluster security group created | `string` | `"EKS cluster security group"` | no | -| [cluster\_security\_group\_id](#input\_cluster\_security\_group\_id) | Existing security group ID to be attached to the cluster | `string` | `""` | no | -| [cluster\_security\_group\_name](#input\_cluster\_security\_group\_name) | Name to use on cluster security group created | `string` | `null` | no | -| [cluster\_security\_group\_tags](#input\_cluster\_security\_group\_tags) | A map of additional tags to add to the cluster security group created | `map(string)` | `{}` | no | -| [cluster\_security\_group\_use\_name\_prefix](#input\_cluster\_security\_group\_use\_name\_prefix) | Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix | `bool` | `true` | no | -| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | -| [cluster\_service\_ipv6\_cidr](#input\_cluster\_service\_ipv6\_cidr) | The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | -| [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | -| [cluster\_upgrade\_policy](#input\_cluster\_upgrade\_policy) | Configuration block for the cluster upgrade policy | `any` | `{}` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.27`) | `string` | `null` | no | -| [cluster\_zonal\_shift\_config](#input\_cluster\_zonal\_shift\_config) | Configuration block for the cluster zonal shift | `any` | `{}` | no | +| [compute\_config](#input\_compute\_config) | Configuration block for the cluster compute configuration |
object({
enabled = optional(bool, false)
node_pools = optional(list(string))
node_role_arn = optional(string)
})
| `null` | no | | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | -| [create\_cluster\_primary\_security\_group\_tags](#input\_create\_cluster\_primary\_security\_group\_tags) | Indicates whether or not to tag the cluster's primary security group. This security group is created by the EKS service, not the module, and therefore tagging is handled after cluster creation | `bool` | `true` | no | -| [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default | `bool` | `true` | no | | [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy) | `bool` | `false` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created for the cluster | `bool` | `true` | no | | [create\_kms\_key](#input\_create\_kms\_key) | Controls if a KMS key for cluster encryption should be created | `bool` | `true` | no | | [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an EKS Auto node IAM role is created | `bool` | `true` | no | | [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | +| [create\_primary\_security\_group\_tags](#input\_create\_primary\_security\_group\_tags) | Indicates whether or not to tag the cluster's primary security group. This security group is created by the EKS service, not the module, and therefore tagging is handled after cluster creation | `bool` | `true` | no | +| [create\_security\_group](#input\_create\_security\_group) | Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default | `bool` | `true` | no | | [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | | [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed node group(s), self-managed node group(s), Fargate profile(s)) | `string` | `"30s"` | no | -| [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | -| [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | +| [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# EKS Managed Node Group
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
ami_id = optional(string)
ami_type = optional(string)
ami_release_version = optional(string)
use_latest_ami_release_version = optional(bool)
capacity_type = optional(string)
disk_size = optional(number)
force_update_version = optional(bool)
instance_types = optional(list(string))
labels = optional(map(string))
node_repair_config = optional(object({
enabled = optional(bool)
}))
remote_access = optional(object({
ec2_ssh_key = optional(string)
source_security_group_ids = optional(list(string))
}))
taints = optional(map(object({
key = string
value = optional(string)
effect = string
})))
update_config = optional(object({
max_unavailable = optional(number)
max_unavailable_percentage = optional(number)
}))
timeouts = optional(object({
create = optional(string)
update = optional(string)
delete = optional(string)
}))
# User data
enable_bootstrap_user_data = optional(bool)
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
key_name = optional(string)
disable_api_termination = optional(bool)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
create_placement_group = optional(bool)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
subnet_id = optional(string)
})))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_role = optional(bool)
iam_role_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
cluster_primary_security_group_id = optional(string)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})), {})
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | | [enable\_auto\_mode\_custom\_tags](#input\_enable\_auto\_mode\_custom\_tags) | Determines whether to enable permissions for custom tags resources created by EKS Auto Mode | `bool` | `true` | no | | [enable\_cluster\_creator\_admin\_permissions](#input\_enable\_cluster\_creator\_admin\_permissions) | Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry | `bool` | `false` | no | -| [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | | [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled | `bool` | `true` | no | -| [enable\_security\_groups\_for\_pods](#input\_enable\_security\_groups\_for\_pods) | Determines whether to add the necessary IAM permission policy for security groups for pods | `bool` | `true` | no | -| [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no | -| [fargate\_profiles](#input\_fargate\_profiles) | Map of Fargate Profile definitions to create | `any` | `{}` | no | +| [enabled\_log\_types](#input\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | +| [encryption\_config](#input\_encryption\_config) | Configuration block with encryption configuration for the cluster |
object({
provider_key_arn = optional(string)
resources = optional(list(string), ["secrets"])
})
| `{}` | no | +| [encryption\_policy\_description](#input\_encryption\_policy\_description) | Description of the cluster encryption policy created | `string` | `"Cluster encryption policy to allow cluster role to utilize CMK provided"` | no | +| [encryption\_policy\_name](#input\_encryption\_policy\_name) | Name to use on cluster encryption policy created | `string` | `null` | no | +| [encryption\_policy\_path](#input\_encryption\_policy\_path) | Cluster encryption policy path | `string` | `null` | no | +| [encryption\_policy\_tags](#input\_encryption\_policy\_tags) | A map of additional tags to add to the cluster encryption policy created | `map(string)` | `{}` | no | +| [encryption\_policy\_use\_name\_prefix](#input\_encryption\_policy\_use\_name\_prefix) | Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix | `bool` | `true` | no | +| [endpoint\_private\_access](#input\_endpoint\_private\_access) | Indicates whether or not the Amazon EKS private API server endpoint is enabled | `bool` | `true` | no | +| [endpoint\_public\_access](#input\_endpoint\_public\_access) | Indicates whether or not the Amazon EKS public API server endpoint is enabled | `bool` | `false` | no | +| [endpoint\_public\_access\_cidrs](#input\_endpoint\_public\_access\_cidrs) | List of CIDR blocks which can access the Amazon EKS public API server endpoint | `list(string)` |
[
"0.0.0.0/0"
]
| no | +| [fargate\_profiles](#input\_fargate\_profiles) | Map of Fargate Profile definitions to create |
map(object({
create = optional(bool)

# Fargate profile
name = optional(string) # Will fall back to map key
subnet_ids = optional(list(string))
selectors = optional(list(object({
labels = optional(map(string))
namespace = string
})))
timeouts = optional(object({
create = optional(string)
delete = optional(string)
}))

# IAM role
create_iam_role = optional(bool)
iam_role_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
tags = optional(map(string))
}))
| `null` | no | +| [force\_update\_version](#input\_force\_update\_version) | Force version update by overriding upgrade-blocking readiness checks when updating a cluster | `bool` | `null` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the cluster. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | @@ -452,7 +430,9 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [identity\_providers](#input\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA |
map(object({
client_id = string
groups_claim = optional(string)
groups_prefix = optional(string)
identity_provider_config_name = optional(string) # will fall back to map key
issuer_url = string
required_claims = optional(map(string))
username_claim = optional(string)
username_prefix = optional(string)
}))
| `null` | no | | [include\_oidc\_root\_ca\_thumbprint](#input\_include\_oidc\_root\_ca\_thumbprint) | Determines whether to include the root CA thumbprint in the OpenID Connect (OIDC) identity provider's server certificate(s) | `bool` | `true` | no | +| [ip\_family](#input\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `"ipv4"` | no | | [kms\_key\_administrators](#input\_kms\_key\_administrators) | A list of IAM ARNs for [key administrators](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-administrators). If no value is provided, the current caller identity is used to ensure at least one key admin is available | `list(string)` | `[]` | no | | [kms\_key\_aliases](#input\_kms\_key\_aliases) | A list of aliases to create. Note - due to the use of `toset()`, values must be static strings and not computed values | `list(string)` | `[]` | no | | [kms\_key\_deletion\_window\_in\_days](#input\_kms\_key\_deletion\_window\_in\_days) | The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key. If you specify a value, it must be between `7` and `30`, inclusive. If you do not specify a value, it defaults to `30` | `number` | `null` | no | @@ -463,6 +443,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [kms\_key\_service\_users](#input\_kms\_key\_service\_users) | A list of IAM ARNs for [key service users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-service-integration) | `list(string)` | `[]` | no | | [kms\_key\_source\_policy\_documents](#input\_kms\_key\_source\_policy\_documents) | List of IAM policy documents that are merged together into the exported document. Statements must have unique `sid`s | `list(string)` | `[]` | no | | [kms\_key\_users](#input\_kms\_key\_users) | A list of IAM ARNs for [key users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-users) | `list(string)` | `[]` | no | +| [kubernetes\_version](#input\_kubernetes\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.33`) | `string` | `null` | no | +| [name](#input\_name) | Name of the EKS cluster | `string` | `""` | no | | [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the EKS Auto node IAM role | `map(string)` | `{}` | no | | [node\_iam\_role\_description](#input\_node\_iam\_role\_description) | Description of the EKS Auto node IAM role | `string` | `null` | no | | [node\_iam\_role\_name](#input\_node\_iam\_role\_name) | Name to use on the EKS Auto node IAM role created | `string` | `null` | no | @@ -470,7 +452,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [node\_iam\_role\_permissions\_boundary](#input\_node\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the EKS Auto node IAM role | `string` | `null` | no | | [node\_iam\_role\_tags](#input\_node\_iam\_role\_tags) | A map of additional tags to add to the EKS Auto node IAM role created | `map(string)` | `{}` | no | | [node\_iam\_role\_use\_name\_prefix](#input\_node\_iam\_role\_use\_name\_prefix) | Determines whether the EKS Auto node IAM role name (`node_iam_role_name`) is used as a prefix | `bool` | `true` | no | -| [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no | +| [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source |
map(object({
protocol = optional(string, "tcp")
from_port = number
to_port = number
type = optional(string, "ingress")
description = optional(string)
cidr_blocks = optional(list(string))
ipv6_cidr_blocks = optional(list(string))
prefix_list_ids = optional(list(string))
self = optional(bool)
source_cluster_security_group = optional(bool, false)
source_security_group_id = optional(string)
}))
| `{}` | no | | [node\_security\_group\_description](#input\_node\_security\_group\_description) | Description of the node security group created | `string` | `"EKS node shared security group"` | no | | [node\_security\_group\_enable\_recommended\_rules](#input\_node\_security\_group\_enable\_recommended\_rules) | Determines whether to enable recommended security group rules for the node security group created. This includes node-to-node TCP ingress on ephemeral ports and allows all egress traffic | `bool` | `true` | no | | [node\_security\_group\_id](#input\_node\_security\_group\_id) | ID of an existing security group to attach to the node groups created | `string` | `""` | no | @@ -478,14 +460,26 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [node\_security\_group\_tags](#input\_node\_security\_group\_tags) | A map of additional tags to add to the node security group created | `map(string)` | `{}` | no | | [node\_security\_group\_use\_name\_prefix](#input\_node\_security\_group\_use\_name\_prefix) | Determines whether node security group name (`node_security_group_name`) is used as a prefix | `bool` | `true` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | List of OpenID Connect audience client IDs to add to the IRSA provider | `list(string)` | `[]` | no | -| [outpost\_config](#input\_outpost\_config) | Configuration for the AWS Outpost to provision the cluster on | `any` | `{}` | no | +| [outpost\_config](#input\_outpost\_config) | Configuration for the AWS Outpost to provision the cluster on |
object({
control_plane_instance_type = optional(string)
control_plane_placement = optional(object({
group_name = string
}))
outpost_arns = list(string)
})
| `null` | no | | [prefix\_separator](#input\_prefix\_separator) | The separator to use between the prefix and the generated timestamp for resource names | `string` | `"-"` | no | | [putin\_khuylo](#input\_putin\_khuylo) | Do you agree that Putin doesn't respect Ukrainian sovereignty and territorial integrity? More info: https://en.wikipedia.org/wiki/Putin_khuylo! | `bool` | `true` | no | -| [self\_managed\_node\_group\_defaults](#input\_self\_managed\_node\_group\_defaults) | Map of self-managed node group default configurations | `any` | `{}` | no | -| [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create | `any` | `{}` | no | +| [region](#input\_region) | Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration | `string` | `null` | no | +| [remote\_network\_config](#input\_remote\_network\_config) | Configuration block for the cluster remote network configuration |
object({
remote_node_networks = object({
cidrs = optional(list(string))
})
remote_pod_networks = optional(object({
cidrs = optional(list(string))
}))
})
| `null` | no | +| [security\_group\_additional\_rules](#input\_security\_group\_additional\_rules) | List of additional security group rules to add to the cluster security group created. Set `source_node_security_group = true` inside rules to set the `node_security_group` as source |
map(object({
protocol = optional(string, "tcp")
from_port = number
to_port = number
type = optional(string, "ingress")
description = optional(string)
cidr_blocks = optional(list(string))
ipv6_cidr_blocks = optional(list(string))
prefix_list_ids = optional(list(string))
self = optional(bool)
source_node_security_group = optional(bool, false)
source_security_group_id = optional(string)
}))
| `{}` | no | +| [security\_group\_description](#input\_security\_group\_description) | Description of the cluster security group created | `string` | `"EKS cluster security group"` | no | +| [security\_group\_id](#input\_security\_group\_id) | Existing security group ID to be attached to the cluster | `string` | `""` | no | +| [security\_group\_name](#input\_security\_group\_name) | Name to use on cluster security group created | `string` | `null` | no | +| [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the cluster security group created | `map(string)` | `{}` | no | +| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix | `bool` | `true` | no | +| [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create |
map(object({
create = optional(bool, true)
# Autoscaling Group
create_autoscaling_group = optional(bool)
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
availability_zones = optional(list(string))
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
desired_size_type = optional(string)
capacity_rebalance = optional(bool)
default_instance_warmup = optional(number)
protect_from_scale_in = optional(bool)
context = optional(string)
create_placement_group = optional(bool)
placement_group = optional(string)
health_check_type = optional(string)
health_check_grace_period = optional(number)
ignore_failed_scaling_activities = optional(bool)
force_delete = optional(bool)
termination_policies = optional(list(string))
suspended_processes = optional(list(string))
max_instance_lifetime = optional(number)
enabled_metrics = optional(list(string))
metrics_granularity = optional(string)
initial_lifecycle_hooks = optional(list(object({
default_result = optional(string)
heartbeat_timeout = optional(number)
lifecycle_transition = string
name = string
notification_metadata = optional(string)
notification_target_arn = optional(string)
role_arn = optional(string)
})))
instance_maintenance_policy = optional(object({
max_healthy_percentage = number
min_healthy_percentage = number
}))
instance_refresh = optional(object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string)
triggers = optional(list(string))
}))
use_mixed_instances_policy = optional(bool)
mixed_instances_policy = optional(object({
instances_distribution = optional(object({
on_demand_allocation_strategy = optional(string)
on_demand_base_capacity = optional(number)
on_demand_percentage_above_base_capacity = optional(number)
spot_allocation_strategy = optional(string)
spot_instance_pools = optional(number)
spot_max_price = optional(string)
}))
launch_template = object({
override = optional(list(object({
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = optional(number)
}))
}))
instance_type = optional(string)
launch_template_specification = optional(object({
launch_template_id = optional(string)
launch_template_name = optional(string)
version = optional(string)
}))
weighted_capacity = optional(string)
})))
})
}))
timeouts = optional(object({
delete = optional(string)
}))
autoscaling_group_tags = optional(map(string))
# User data
ami_type = optional(string)
additional_cluster_dns_ips = optional(list(string))
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
ami_id = optional(string)
instance_type = optional(string)
key_name = optional(string)
disable_api_termination = optional(bool)
instance_initiated_shutdown_behavior = optional(string)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = string
}))
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string))
subnet_id = optional(string)
})))
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_instance_profile = optional(bool)
iam_instance_profile_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Access entry
create_access_entry = optional(bool)
iam_role_arn = optional(string)
# Security group
attach_cluster_primary_security_group = optional(bool, false)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | +| [service\_ipv4\_cidr](#input\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | +| [service\_ipv6\_cidr](#input\_service\_ipv6\_cidr) | The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster | `string` | `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets | `list(string)` | `[]` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | +| [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the cluster |
object({
create = optional(string)
update = optional(string)
delete = optional(string)
})
| `null` | no | +| [upgrade\_policy](#input\_upgrade\_policy) | Configuration block for the cluster upgrade policy |
object({
support_type = optional(string)
})
| `null` | no | | [vpc\_id](#input\_vpc\_id) | ID of the VPC where the cluster security group will be provisioned | `string` | `null` | no | +| [zonal\_shift\_config](#input\_zonal\_shift\_config) | Configuration block for the cluster zonal shift |
object({
enabled = optional(bool)
})
| `null` | no | ## Outputs diff --git a/docs/README.md b/docs/README.md index 144826b4cf..960db18448 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,3 +11,4 @@ - [Upgrade to v18.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-18.0.md) - [Upgrade to v19.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-19.0.md) - [Upgrade to v20.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-20.0.md) + - [Upgrade to v21.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-21.0.md) diff --git a/docs/UPGRADE-21.0.md b/docs/UPGRADE-21.0.md new file mode 100644 index 0000000000..695c8c80f2 --- /dev/null +++ b/docs/UPGRADE-21.0.md @@ -0,0 +1,328 @@ +# Upgrade from v20.x to v21.x + +If you have any questions regarding this upgrade process, please consult the [`examples`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples) directory: +If you find a bug, please open an issue with supporting configuration to reproduce. + +## List of backwards incompatible changes + +- Terraform `v1.5.7` is now minimum supported version +- AWS provider `v6.0.0` is now minimum supported version +- TLS provider `v4.0.0` is now minimum supported version +- The `aws-auth` sub-module has been removed. Users who wish to utilize its functionality can continue to do so by specifying a `v20.x` version, or `~> v20.0` version constraint in their module source. +- `bootstrap_self_managed_addons` is now hardcoded to `false`. This is a legacy setting and instead users should utilize the EKS addons API, which is what this module does by default. In conjunction with this change, the `bootstrap_self_managed_addons` is now ignored by the module to aid in upgrading without disruption (otherwise it would require cluster re-creation). +- When enabling `enable_efa_support` or creating placement groups within a node group, users must now specify the correct `subnet_ids`; the module no longer tries to automatically select a suitable subnet. +- EKS managed node group: + - IMDS now default to a hop limit of 1 (previously was 2) + - `ami_type` now defaults to `AL2023_x86_64_STANDARD` + - `enable_monitoring` is now set to `false` by default + - `enable_efa_only` is now set to `true` by default + - `use_latest_ami_release_version` is now set to `true` by default + - Support for autoscaling group schedules has been removed +- Self-managed node group: + - IMDS now default to a hop limit of 1 (previously was 2) + - `ami_type` now defaults to `AL2023_x86_64_STANDARD` + - `enable_monitoring` is now set to `false` by default + - `enable_efa_only` is now set to `true` by default + - Support for autoscaling group schedules has been removed +- Karpenter: + - Native support for IAM roles for service accounts (IRSA) has been removed; EKS Pod Identity is now enabled by default + - Karpenter controller policy for prior to Karpenter `v1` have been removed (i.e. `v0.33`); the `v1` policy is now used by default + - `create_pod_identity_association` is now set to `true` by default +- `addons.resolve_conflicts_on_create` is now set to `"NONE"` by default (was `"OVERWRITE"`). +- `addons.most_recent` is now set to `true` by default (was `false`). +- `cluster_identity_providers.issuer_url` is now required to be set by users; the prior incorrect default has been removed. See https://github.com/terraform-aws-modules/terraform-aws-eks/pull/3055 and https://github.com/kubernetes/kubernetes/pull/123561 for more details. +- The OIDC issuer URL for IAM roles for service accounts (IRSA) has been changed to use the new dual stack`oidc-eks` endpoint instead of `oidc.eks`. This is to align with https://github.com/aws/containers-roadmap/issues/2038#issuecomment-2278450601 + +## Additional changes + +### Added + +- Support for `region` parameter to specify the AWS region for the resources created if different from the provider region. +- Both the EKS managed and self-managed node groups now support creating their own security groups (again). This is primarily motivated by the changes for EFA support; previously users would need to specify `enable_efa_support` both at the cluster level (to add the appropriate security group rules to the shared node security group) as well as the node group level. However, its not always desirable to have these rules across ALL node groups when they are really only required on the node group where EFA is utilized. And similarly for other use cases, users can create custom rules for a specific node group instead of apply across ALL node groups. + +### Modified + +- Variable definitions now contain detailed `object` types in place of the previously used any type. +- The embedded KMS key module definition has been updated to `v4.0` to support the same version requirements as well as the new `region` argument. + +### Variable and output changes + +1. Removed variables: + + - `enable_efa_support` - users only need to set this within the node group configuration, as the module no longer manages EFA support at the cluster level. + - `enable_security_groups_for_pods` - users can instead attach the `arn:aws:iam::aws:policy/AmazonEKSVPCResourceController` policy via `iam_role_additional_policies` if using security groups for pods. + - `eks-managed-node-group` sub-module + - `cluster_service_ipv4_cidr` - users should use `cluster_service_cidr` instead (for either IPv4 or IPv6). + - `elastic_gpu_specifications` + - `elastic_inference_accelerator` + - `platform` - this is superseded by `ami_type` + - `placement_group_strategy` - set to `cluster` by the module + - `placement_group_az` - users will need to specify the correct subnet in `subnet_ids` + - `create_schedule` + - `schedules` + - `self-managed-node-group` sub-module + - `elastic_gpu_specifications` + - `elastic_inference_accelerator` + - `platform` - this is superseded by `ami_type` + - `create_schedule` + - `schedules` + - `placement_group_az` - users will need to specify the correct subnet in `subnet_ids` + - `hibernation_options` - not valid in EKS + - `min_elb_capacity` - not valid in EKS + - `wait_for_elb_capacity` - not valid in EKS + - `wait_for_capacity_timeout` - not valid in EKS + - `default_cooldown` - not valid in EKS + - `target_group_arns` - not valid in EKS + - `service_linked_role_arn` - not valid in EKS + - `warm_pool` - not valid in EKS + - `fargate-profile` sub-module + - None + - `karpenter` sub-module + - `enable_v1_permissions` - v1 permissions are now the default + - `enable_irsa` + - `irsa_oidc_provider_arn` + - `irsa_namespace_service_accounts` + - `irsa_assume_role_condition_test` + +2. Renamed variables: + + - Variables prefixed with `cluster_*` have been stripped of the prefix to better match the underlying API: + - `cluster_name` -> `name` + - `cluster_version` -> `kubernetes_version` + - `cluster_enabled_log_types` -> `enabled_log_types` + - `cluster_force_update_version` -> `force_update_version` + - `cluster_compute_config` -> `compute_config` + - `cluster_upgrade_policy` -> `upgrade_policy` + - `cluster_remote_network_config` -> `remote_network_config` + - `cluster_zonal_shift_config` -> `zonal_shift_config` + - `cluster_additional_security_group_ids` -> `additional_security_group_ids` + - `cluster_endpoint_private_access` -> `endpoint_private_access` + - `cluster_endpoint_public_access` -> `endpoint_public_access` + - `cluster_endpoint_public_access_cidrs` -> `endpoint_public_access_cidrs` + - `cluster_ip_family` -> `ip_family` + - `cluster_service_ipv4_cidr` -> `service_ipv4_cidr` + - `cluster_service_ipv6_cidr` -> `service_ipv6_cidr` + - `cluster_encryption_config` -> `encryption_config` + - `create_cluster_primary_security_group_tags` -> `create_primary_security_group_tags` + - `cluster_timeouts` -> `timeouts` + - `create_cluster_security_group` -> `create_security_group` + - `cluster_security_group_id` -> `security_group_id` + - `cluster_security_group_name` -> `security_group_name` + - `cluster_security_group_use_name_prefix` -> `security_group_use_name_prefix` + - `cluster_security_group_description` -> `security_group_description` + - `cluster_security_group_additional_rules` -> `security_group_additional_rules` + - `cluster_security_group_tags` -> `security_group_tags` + - `cluster_encryption_policy_use_name_prefix` -> `encryption_policy_use_name_prefix` + - `cluster_encryption_policy_name` -> `encryption_policy_name` + - `cluster_encryption_policy_description` -> `encryption_policy_description` + - `cluster_encryption_policy_path` -> `encryption_policy_path` + - `cluster_encryption_policy_tags` -> `encryption_policy_tags` + - `cluster_addons` -> `addons` + - `cluster_addons_timeouts` -> `addons_timeouts` + - `cluster_identity_providers` -> `identity_providers` + - `eks-managed-node-group` sub-module + - `cluster_version` -> `kubernetes_version` + - `self-managed-node-group` sub-module + - `cluster_version` -> `kubernetes_version` + - `delete_timeout` -> `timeouts` + - `fargate-profile` sub-module + - None + - `karpenter` sub-module + - None + +3. Added variables: + + - `region` + - `eks-managed-node-group` sub-module + - `region` + - `partition` - added to reduce number of `GET` requests from data sources when possible + - `account_id` - added to reduce number of `GET` requests from data sources when possible + - `create_security_group` + - `security_group_name` + - `security_group_use_name_prefix` + - `security_group_description` + - `security_group_ingress_rules` + - `security_group_egress_rules` + - `security_group_tags` + - `self-managed-node-group` sub-module + - `region` + - `partition` - added to reduce number of `GET` requests from data sources when possible + - `account_id` - added to reduce number of `GET` requests from data sources when possible + - `create_security_group` + - `security_group_name` + - `security_group_use_name_prefix` + - `security_group_description` + - `security_group_ingress_rules` + - `security_group_egress_rules` + - `security_group_tags` + - `fargate-profile` sub-module + - `region` + - `partition` - added to reduce number of `GET` requests from data sources when possible + - `account_id` - added to reduce number of `GET` requests from data sources when possible + - `karpenter` sub-module + - `region` + +4. Removed outputs: + + - `eks-managed-node-group` sub-module + - `platform` - this is superseded by `ami_type` + - `autoscaling_group_schedule_arns` + - `self-managed-node-group` sub-module + - `platform` - this is superseded by `ami_type` + - `autoscaling_group_schedule_arns` + - `fargate-profile` sub-module + - None + - `karpenter` sub-module + - None + +5. Renamed outputs: + + - `eks-managed-node-group` sub-module + - None + - `self-managed-node-group` sub-module + - None + - `fargate-profile` sub-module + - None + - `karpenter` sub-module + - None + +6. Added outputs: + + - `eks-managed-node-group` sub-module + - `security_group_arn` + - `security_group_id` + - `self-managed-node-group` sub-module + - `security_group_arn` + - `security_group_id` + - `fargate-profile` sub-module + - None + - `karpenter` sub-module + - None + +## Upgrade Migrations + +### Before 20.x Example + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + # Truncated for brevity ... + # Renamed variables are not shown here, please refer to the full list above. + + enable_efa_support = true + + eks_managed_node_group_defaults = { + iam_role_additional_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + } + + eks_managed_node_groups = { + efa = { + ami_type = "AL2023_x86_64_NVIDIA" + instance_types = ["p5e.48xlarge"] + + enable_efa_support = true + enable_efa_only = true + } + } + + self_managed_node_groups = { + example = { + use_mixed_instances_policy = true + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 0 + on_demand_allocation_strategy = "lowest-price" + spot_allocation_strategy = "price-capacity-optimized" + } + + # ASG configuration + override = [ + { + instance_requirements = { + cpu_manufacturers = ["intel"] + instance_generations = ["current", "previous"] + spot_max_price_percentage_over_lowest_price = 100 + + vcpu_count = { + min = 1 + } + + allowed_instance_types = ["t*", "m*"] + } + } + ] + } + } + } +} +``` + +### After 21.x Example + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" + + # Truncated for brevity ... + # Renamed variables are not shown here, please refer to the full list above. + + eks_managed_node_groups = { + efa = { + ami_type = "AL2023_x86_64_NVIDIA" + instance_types = ["p5e.48xlarge"] + + iam_role_additional_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + + enable_efa_support = true + + subnet_ids = element(module.vpc.private_subnets, 0) + } + } + + self_managed_node_groups = { + example = { + use_mixed_instances_policy = true + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 0 + on_demand_allocation_strategy = "lowest-price" + spot_allocation_strategy = "price-capacity-optimized" + } + + # ASG configuration + # Need to wrap in `launch_template` now + launch_template = { + override = [ + { + instance_requirements = { + cpu_manufacturers = ["intel"] + instance_generations = ["current", "previous"] + spot_max_price_percentage_over_lowest_price = 100 + + vcpu_count = { + min = 1 + } + + allowed_instance_types = ["t*", "m*"] + } + } + ] + } + } + } + } +} +``` + +### State Changes + +No state changes required. diff --git a/docs/compute_resources.md b/docs/compute_resources.md index a7bd5b5e93..e7dee7660c 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -56,22 +56,34 @@ Refer to the [EKS Managed Node Group documentation](https://docs.aws.amazon.com/ ```hcl eks_managed_node_groups = { custom_ami = { - ami_id = "ami-0caf35bc73450c396" + ami_id = "ami-0caf35bc73450c396" + ami_type = "AL2023_x86_64_STANDARD" # By default, EKS managed node groups will not append bootstrap script; # this adds it back in using the default template provided by the module # Note: this assumes the AMI provided is an EKS optimized AMI derivative enable_bootstrap_user_data = true - pre_bootstrap_user_data = <<-EOT - export FOO=bar - EOT - - # Because we have full control over the user data supplied, we can also run additional - # scripts/configuration changes after the bootstrap script has been run - post_bootstrap_user_data = <<-EOT - echo "you are free little kubelet!" - EOT + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + EOT + content_type = "application/node.eks.aws" + }] + + # This is only possible when `ami_id` is specified, indicating a custom AMI + cloudinit_post_nodeadm = [{ + content = <<-EOT + echo "All done" + EOT + content_type = "text/x-shellscript; charset=\"us-ascii\"" + }] } } ``` @@ -113,9 +125,9 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 1. The `self-managed-node-group` uses the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version by default: ```hcl - cluster_version = "1.33" + kubernetes_version = "1.33" - # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.27 + # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.33 self_managed_node_groups = { default = {} } @@ -124,7 +136,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 2. To use Bottlerocket, specify the `ami_type` as one of the respective `"BOTTLEROCKET_*" types` and supply a Bottlerocket OS AMI: ```hcl - cluster_version = "1.33" + kubernetes_version = "1.33" self_managed_node_groups = { bottlerocket = { @@ -139,42 +151,3 @@ See the [`examples/self-managed-node-group/` example](https://github.com/terrafo ### Fargate Profiles Fargate profiles are straightforward to use and therefore no further details are provided here. See the [`tests/fargate-profile/` tests](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/tests/fargate-profile) for a working example of various configurations. - -### Default Configurations - -Each type of compute resource (EKS managed node group, self managed node group, or Fargate profile) provides the option for users to specify a default configuration. These default configurations can be overridden from within the compute resource's individual definition. The order of precedence for configurations (from highest to least precedence): - -- Compute resource individual configuration - - Compute resource family default configuration (`eks_managed_node_group_defaults`, `self_managed_node_group_defaults`, `fargate_profile_defaults`) - - Module default configuration (see `variables.tf` and `node_groups.tf`) - -For example, the following creates 4 AWS EKS Managed Node Groups: - -```hcl - eks_managed_node_group_defaults = { - ami_type = "AL2_x86_64" - disk_size = 50 - instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] - } - - eks_managed_node_groups = { - # Uses module default configurations overridden by configuration above - default = {} - - # This further overrides the instance types used - compute = { - instance_types = ["c5.large", "c6i.large", "c6d.large"] - } - - # This further overrides the instance types and disk size used - persistent = { - disk_size = 1024 - instance_types = ["r5.xlarge", "r6i.xlarge", "r5b.xlarge"] - } - - # This overrides the OS used - bottlerocket = { - ami_type = "BOTTLEROCKET_x86_64" - } - } -``` diff --git a/docs/faq.md b/docs/faq.md index 2abd6695c2..69d8c90e7f 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -12,23 +12,44 @@ `disk_size`, and `remote_access` can only be set when using the EKS managed node group default launch template. This module defaults to providing a custom launch template to allow for custom security groups, tag propagation, etc. If you wish to forgo the custom launch template route, you can set `use_custom_launch_template = false` and then you can set `disk_size` and `remote_access`. -### I received an error: `expect exactly one securityGroup tagged with kubernetes.io/cluster/ ...` +### I received an error: `expect exactly one securityGroup tagged with kubernetes.io/cluster/ ...` -By default, EKS creates a cluster primary security group that is created outside of the module and the EKS service adds the tag `{ "kubernetes.io/cluster/" = "owned" }`. This on its own does not cause any conflicts for addons such as the AWS Load Balancer Controller until users decide to attach both the cluster primary security group and the shared node security group created by the module (by setting `attach_cluster_primary_security_group = true`). The issue is not with having multiple security groups in your account with this tag key:value combination, but having multiple security groups with this tag key:value combination attached to nodes in the same cluster. There are a few ways to resolve this depending on your use case/intentions: +⚠️ `` would be the name of your cluster -⚠️ `` below needs to be replaced with the name of your cluster +By default, EKS creates a cluster primary security group that is created outside of the module and the EKS service adds the tag `{ "kubernetes.io/cluster/" = "owned" }`. This on its own does not cause any conflicts for addons such as the AWS Load Balancer Controller until users decide to attach both the cluster primary security group and the shared node security group created by the module (by setting `attach_cluster_primary_security_group = true`). The issue is not with having multiple security groups in your account with this tag key:value combination, but having multiple security groups with this tag key:value combination attached to nodes in the same cluster. There are a few ways to resolve this depending on your use case/intentions: 1. If you want to use the cluster primary security group, you can disable the creation of the shared node security group with: ```hcl - create_node_security_group = false # default is true - attach_cluster_primary_security_group = true # default is false + create_node_security_group = false # default is true + + eks_managed_node_group = { + example = { + attach_cluster_primary_security_group = true # default is false + } + } + # Or for self-managed + self_managed_node_group = { + example = { + attach_cluster_primary_security_group = true # default is false + } + } ``` 2. By not attaching the cluster primary security group. The cluster primary security group has quite broad access and the module has instead provided a security group with the minimum amount of access to launch an empty EKS cluster successfully and users are encouraged to open up access when necessary to support their workload. ```hcl - attach_cluster_primary_security_group = false # this is the default for the module + eks_managed_node_group = { + example = { + attach_cluster_primary_security_group = true # default is false + } + } + # Or for self-managed + self_managed_node_group = { + example = { + attach_cluster_primary_security_group = true # default is false + } + } ``` In theory, if you are attaching the cluster primary security group, you shouldn't need to use the shared node security group created by the module. However, this is left up to users to decide for their requirements and use case. @@ -58,6 +79,8 @@ If you require a public endpoint, setting up both (public and private) and restr The module is configured to ignore this value. Unfortunately, Terraform does not support variables within the `lifecycle` block. The setting is ignored to allow autoscaling via controllers such as cluster autoscaler or Karpenter to work properly and without interference by Terraform. Changing the desired count must be handled outside of Terraform once the node group is created. +:info: See [this](https://github.com/bryantbiggs/eks-desired-size-hack) for a workaround to this limitation. + ### How do I access compute resource attributes? Examples of accessing the attributes of the compute resource(s) created by the root module are shown below. Note - the assumption is that your cluster module definition is named `eks` as in `module "eks" { ... }`: @@ -90,6 +113,10 @@ aws eks describe-addon-versions --query 'addons[*].addonName' ### What configuration values are available for an add-on? +> [!NOTE] +> The available configuration values will vary between add-on versions, +> typically more configuration values will be added in later versions as functionality is enabled by EKS. + You can retrieve the configuration value schema for a given addon using the following command: ```sh @@ -286,7 +313,3 @@ Returns (at the time of writing): } } ``` - -> [!NOTE] -> The available configuration values will vary between add-on versions, -> typically more configuration values will be added in later versions as functionality is enabled by EKS. diff --git a/docs/network_connectivity.md b/docs/network_connectivity.md index 9d38fc130d..2cb38d5e3e 100644 --- a/docs/network_connectivity.md +++ b/docs/network_connectivity.md @@ -27,7 +27,7 @@ See the example snippet below which adds additional security group rules to the ```hcl ... # Extend cluster security group rules - cluster_security_group_additional_rules = { + security_group_additional_rules = { egress_nodes_ephemeral_ports_tcp = { description = "To node 1025-65535" protocol = "tcp" diff --git a/docs/user_data.md b/docs/user_data.md index 3ca2263deb..2bd17f69a9 100644 --- a/docs/user_data.md +++ b/docs/user_data.md @@ -10,7 +10,8 @@ Users can see the various methods of using and providing user data through the [ - AMI types of `BOTTLEROCKET_*`, user data must be in TOML format - AMI types of `WINDOWS_*`, user data must be in powershell/PS1 script format - Self Managed Node Groups - - `AL2_x86_64` AMI type (default) -> the user data template (bash/shell script) provided by the module is used as the default; users are able to provide their own user data template + - `AL2_*` AMI types -> the user data template (bash/shell script) provided by the module is used as the default; users are able to provide their own user data template + - `AL2023_*` AMI types -> the user data template (MIME multipart format) provided by the module is used as the default; users are able to provide their own user data template - `BOTTLEROCKET_*` AMI types -> the user data template (TOML file) provided by the module is used as the default; users are able to provide their own user data template - `WINDOWS_*` AMI types -> the user data template (powershell/PS1 script) provided by the module is used as the default; users are able to provide their own user data template @@ -24,9 +25,24 @@ When using an EKS managed node group, users have 2 primary routes for interactin - Users can use the following variables to facilitate this process: - ```hcl - pre_bootstrap_user_data = "..." - ``` + For `AL2_*`, `BOTTLEROCKET_*`, and `WINDOWS_*`: + ```hcl + pre_bootstrap_user_data = "..." + ``` + + For `AL2023_*` + ```hcl + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + ... + EOT + content_type = "application/node.eks.aws" + }] + ``` 2. If a custom AMI is used, then per the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami), users will need to supply the necessary user data to bootstrap and register nodes with the cluster when launched. There are two routes to facilitate this bootstrapping process: - If the AMI used is a derivative of the [AWS EKS Optimized AMI ](https://github.com/awslabs/amazon-eks-ami), users can opt in to using a template provided by the module that provides the minimum necessary configuration to bootstrap the node when launched: diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index 085533d59d..57da18f875 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -24,14 +24,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws](#provider\_aws) | >= 6.0 | ## Modules @@ -39,7 +39,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|--------|---------| | [disabled\_eks](#module\_disabled\_eks) | ../.. | n/a | | [eks](#module\_eks) | ../.. | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/eks-auto-mode/main.tf b/examples/eks-auto-mode/main.tf index 316b2764d4..c99c60c970 100644 --- a/examples/eks-auto-mode/main.tf +++ b/examples/eks-auto-mode/main.tf @@ -11,9 +11,9 @@ data "aws_availability_zones" "available" { } locals { - name = "ex-${basename(path.cwd)}" - cluster_version = "1.33" - region = "us-west-2" + name = "ex-${basename(path.cwd)}" + kubernetes_version = "1.33" + region = "us-west-2" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -32,13 +32,13 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_public_access = true + name = local.name + kubernetes_version = local.kubernetes_version + endpoint_public_access = true enable_cluster_creator_admin_permissions = true - cluster_compute_config = { + compute_config = { enabled = true node_pools = ["general-purpose"] } @@ -61,7 +61,7 @@ module "disabled_eks" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index ed8825f17b..94ccb4eb4d 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -25,9 +25,9 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | -| [helm](#requirement\_helm) | >= 2.7, < 3.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | +| [helm](#requirement\_helm) | >= 3.0 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | | [tls](#requirement\_tls) | >= 4.0 | @@ -36,9 +36,9 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | -| [aws.remote](#provider\_aws.remote) | >= 5.95, < 6.0.0 | -| [helm](#provider\_helm) | >= 2.7, < 3.0.0 | +| [aws](#provider\_aws) | >= 6.0 | +| [aws.remote](#provider\_aws.remote) | >= 6.0 | +| [helm](#provider\_helm) | >= 3.0 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | @@ -49,8 +49,8 @@ Note that this example may create resources which cost money. Run `terraform des | [eks](#module\_eks) | ../.. | n/a | | [eks\_hybrid\_node\_role](#module\_eks\_hybrid\_node\_role) | ../../modules/hybrid-node-role | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | -| [remote\_node\_vpc](#module\_remote\_node\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [remote\_node\_vpc](#module\_remote\_node\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/eks-hybrid-nodes/main.tf b/examples/eks-hybrid-nodes/main.tf index ffc77e36a2..d23f05e370 100644 --- a/examples/eks-hybrid-nodes/main.tf +++ b/examples/eks-hybrid-nodes/main.tf @@ -3,11 +3,11 @@ provider "aws" { } provider "helm" { - kubernetes { + kubernetes = { host = module.eks.cluster_endpoint cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - exec { + exec = { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed @@ -20,7 +20,7 @@ locals { name = "ex-${basename(path.cwd)}" region = "us-west-2" - cluster_version = "1.33" + kubernetes_version = "1.33" tags = { Test = local.name @@ -36,20 +36,20 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version + name = local.name + kubernetes_version = local.kubernetes_version - cluster_endpoint_public_access = true + endpoint_public_access = true enable_cluster_creator_admin_permissions = true - cluster_addons = { + addons = { coredns = {} eks-pod-identity-agent = {} kube-proxy = {} } create_node_security_group = false - cluster_security_group_additional_rules = { + security_group_additional_rules = { hybrid-all = { cidr_blocks = [local.remote_network_cidr] description = "Allow all traffic from remote node/pod network" @@ -60,7 +60,7 @@ module "eks" { } } - cluster_compute_config = { + compute_config = { enabled = true node_pools = ["system"] } @@ -75,7 +75,7 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets - cluster_remote_network_config = { + remote_network_config = { remote_node_networks = { cidrs = [local.remote_node_cidr] } @@ -106,7 +106,7 @@ data "aws_availability_zones" "available" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr diff --git a/examples/eks-hybrid-nodes/remote.tf b/examples/eks-hybrid-nodes/remote.tf index 3a44aba523..5aaf85ec96 100644 --- a/examples/eks-hybrid-nodes/remote.tf +++ b/examples/eks-hybrid-nodes/remote.tf @@ -89,7 +89,7 @@ data "aws_ami" "hybrid_node" { provider = aws.remote most_recent = true - name_regex = "eks-hybrid-ubuntu-${local.cluster_version}-amd64-*" + name_regex = "eks-hybrid-ubuntu-${local.kubernetes_version}-amd64-*" owners = ["self"] } @@ -253,7 +253,7 @@ data "aws_availability_zones" "remote" { module "remote_node_vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" providers = { aws = aws.remote diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index 391c421395..29273c9ce0 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -1,14 +1,14 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } helm = { source = "hashicorp/helm" - version = ">= 2.7, < 3.0.0" + version = ">= 3.0" } http = { source = "hashicorp/http" diff --git a/examples/eks-managed-node-group/eks-al2023.tf b/examples/eks-managed-node-group/eks-al2023.tf index 93fda067da..7203c01c81 100644 --- a/examples/eks-managed-node-group/eks-al2023.tf +++ b/examples/eks-managed-node-group/eks-al2023.tf @@ -1,16 +1,21 @@ module "eks_al2023" { - source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" + # source = "terraform-aws-modules/eks/aws" + # version = "~> 20.0" + source = "../.." - cluster_name = "${local.name}-al2023" - cluster_version = "1.33" + name = "${local.name}-al2023" + kubernetes_version = "1.33" # EKS Addons - cluster_addons = { - coredns = {} - eks-pod-identity-agent = {} - kube-proxy = {} - vpc-cni = {} + addons = { + coredns = {} + eks-pod-identity-agent = { + before_compute = true + } + kube-proxy = {} + vpc-cni = { + before_compute = true + } } vpc_id = module.vpc.vpc_id @@ -41,8 +46,6 @@ module "eks_al2023" { kubelet: config: shutdownGracePeriod: 30s - featureGates: - DisableKubeletCloudCredentialProviders: true EOT } ] diff --git a/examples/eks-managed-node-group/eks-bottlerocket.tf b/examples/eks-managed-node-group/eks-bottlerocket.tf index 4d25e5b9c0..2fd3816dbe 100644 --- a/examples/eks-managed-node-group/eks-bottlerocket.tf +++ b/examples/eks-managed-node-group/eks-bottlerocket.tf @@ -1,16 +1,21 @@ module "eks_bottlerocket" { - source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" + # source = "terraform-aws-modules/eks/aws" + # version = "~> 20.0" + source = "../.." - cluster_name = "${local.name}-bottlerocket" - cluster_version = "1.33" + name = "${local.name}-bottlerocket" + kubernetes_version = "1.33" # EKS Addons - cluster_addons = { - coredns = {} - eks-pod-identity-agent = {} - kube-proxy = {} - vpc-cni = {} + addons = { + coredns = {} + eks-pod-identity-agent = { + before_compute = true + } + kube-proxy = {} + vpc-cni = { + before_compute = true + } } vpc_id = module.vpc.vpc_id diff --git a/examples/eks-managed-node-group/main.tf b/examples/eks-managed-node-group/main.tf index 451bbdb385..3adeaeb4c0 100644 --- a/examples/eks-managed-node-group/main.tf +++ b/examples/eks-managed-node-group/main.tf @@ -30,7 +30,7 @@ locals { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 7e10796a2d..47cfbf3ae2 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -93,17 +93,17 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | -| [helm](#requirement\_helm) | >= 2.7, < 3.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | +| [helm](#requirement\_helm) | >= 3.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.95, < 6.0.0 | -| [helm](#provider\_helm) | >= 2.7, < 3.0.0 | +| [aws](#provider\_aws) | >= 6.0 | +| [aws.virginia](#provider\_aws.virginia) | >= 6.0 | +| [helm](#provider\_helm) | >= 3.0.0 | ## Modules @@ -112,7 +112,7 @@ Note that this example may create resources which cost money. Run `terraform des | [eks](#module\_eks) | ../.. | n/a | | [karpenter](#module\_karpenter) | ../../modules/karpenter | n/a | | [karpenter\_disabled](#module\_karpenter\_disabled) | ../../modules/karpenter | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 7006740f47..6f652327c3 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -8,11 +8,11 @@ provider "aws" { } provider "helm" { - kubernetes { + kubernetes = { host = module.eks.cluster_endpoint cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - exec { + exec = { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed @@ -54,19 +54,23 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = "1.33" + name = local.name + kubernetes_version = "1.33" # Gives Terraform identity admin access to cluster which will # allow deploying resources (Karpenter) into the cluster enable_cluster_creator_admin_permissions = true - cluster_endpoint_public_access = true + endpoint_public_access = true - cluster_addons = { - coredns = {} - eks-pod-identity-agent = {} - kube-proxy = {} - vpc-cni = {} + addons = { + coredns = {} + eks-pod-identity-agent = { + before_compute = true + } + kube-proxy = {} + vpc-cni = { + before_compute = true + } } vpc_id = module.vpc.vpc_id @@ -106,8 +110,7 @@ module "eks" { module "karpenter" { source = "../../modules/karpenter" - cluster_name = module.eks.cluster_name - enable_v1_permissions = true + cluster_name = module.eks.cluster_name # Name needs to match role name passed to the EC2NodeClass node_iam_role_use_name_prefix = false @@ -140,7 +143,7 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "1.1.1" + version = "1.6.0" wait = false values = [ @@ -164,7 +167,7 @@ resource "helm_release" "karpenter" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index f64d6ce245..4e81032c58 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -1,14 +1,14 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } helm = { source = "hashicorp/helm" - version = ">= 2.7, < 3.0.0" + version = ">= 3.0.0" } } } diff --git a/examples/self-managed-node-group/eks-al2023.tf b/examples/self-managed-node-group/eks-al2023.tf index 58ecb3cf66..e074735ebb 100644 --- a/examples/self-managed-node-group/eks-al2023.tf +++ b/examples/self-managed-node-group/eks-al2023.tf @@ -1,16 +1,21 @@ module "eks_al2023" { - source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" + # source = "terraform-aws-modules/eks/aws" + # version = "~> 20.0" + source = "../.." - cluster_name = "${local.name}-al2023" - cluster_version = "1.33" + name = "${local.name}-al2023" + kubernetes_version = "1.33" # EKS Addons - cluster_addons = { - coredns = {} - eks-pod-identity-agent = {} - kube-proxy = {} - vpc-cni = {} + addons = { + coredns = {} + eks-pod-identity-agent = { + before_compute = true + } + kube-proxy = {} + vpc-cni = { + before_compute = true + } } vpc_id = module.vpc.vpc_id @@ -40,8 +45,6 @@ module "eks_al2023" { kubelet: config: shutdownGracePeriod: 30s - featureGates: - DisableKubeletCloudCredentialProviders: true EOT } ] diff --git a/examples/self-managed-node-group/eks-bottlerocket.tf b/examples/self-managed-node-group/eks-bottlerocket.tf index adb3462b82..6f0a2dc5ef 100644 --- a/examples/self-managed-node-group/eks-bottlerocket.tf +++ b/examples/self-managed-node-group/eks-bottlerocket.tf @@ -1,16 +1,21 @@ module "eks_bottlerocket" { - source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" + # source = "terraform-aws-modules/eks/aws" + # version = "~> 20.0" + source = "../.." - cluster_name = "${local.name}-bottlerocket" - cluster_version = "1.33" + name = "${local.name}-bottlerocket" + kubernetes_version = "1.33" # EKS Addons - cluster_addons = { - coredns = {} - eks-pod-identity-agent = {} - kube-proxy = {} - vpc-cni = {} + addons = { + coredns = {} + eks-pod-identity-agent = { + before_compute = true + } + kube-proxy = {} + vpc-cni = { + before_compute = true + } } vpc_id = module.vpc.vpc_id diff --git a/examples/self-managed-node-group/main.tf b/examples/self-managed-node-group/main.tf index f8efaeb389..903ee577b9 100644 --- a/examples/self-managed-node-group/main.tf +++ b/examples/self-managed-node-group/main.tf @@ -30,7 +30,7 @@ locals { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/main.tf b/main.tf index c10d79608d..94989384e2 100644 --- a/main.tf +++ b/main.tf @@ -18,14 +18,15 @@ data "aws_iam_session_context" "current" { locals { create = var.create && var.putin_khuylo - partition = try(data.aws_partition.current[0].partition, "") + account_id = try(data.aws_caller_identity.current[0].account_id, "") + partition = try(data.aws_partition.current[0].partition, "") - cluster_role = try(aws_iam_role.this[0].arn, var.iam_role_arn) + role_arn = try(aws_iam_role.this[0].arn, var.iam_role_arn) - create_outposts_local_cluster = length(var.outpost_config) > 0 - enable_cluster_encryption_config = length(var.cluster_encryption_config) > 0 && !local.create_outposts_local_cluster + create_outposts_local_cluster = var.outpost_config != null + enable_encryption_config = length(var.encryption_config) > 0 && !local.create_outposts_local_cluster - auto_mode_enabled = try(var.cluster_compute_config.enabled, false) + auto_mode_enabled = try(var.compute_config.enabled, false) } ################################################################################ @@ -35,12 +36,14 @@ locals { resource "aws_eks_cluster" "this" { count = local.create ? 1 : 0 - name = var.cluster_name - role_arn = local.cluster_role - version = var.cluster_version - enabled_cluster_log_types = var.cluster_enabled_log_types - bootstrap_self_managed_addons = local.auto_mode_enabled ? coalesce(var.bootstrap_self_managed_addons, false) : var.bootstrap_self_managed_addons - force_update_version = var.cluster_force_update_version + region = var.region + + name = var.name + role_arn = local.role_arn + version = var.kubernetes_version + enabled_cluster_log_types = var.enabled_log_types + bootstrap_self_managed_addons = false + force_update_version = var.force_update_version access_config { authentication_mode = var.authentication_mode @@ -54,21 +57,21 @@ resource "aws_eks_cluster" "this" { } dynamic "compute_config" { - for_each = length(var.cluster_compute_config) > 0 ? [var.cluster_compute_config] : [] + for_each = var.compute_config != null ? [var.compute_config] : [] content { - enabled = local.auto_mode_enabled - node_pools = local.auto_mode_enabled ? try(compute_config.value.node_pools, []) : null - node_role_arn = local.auto_mode_enabled && length(try(compute_config.value.node_pools, [])) > 0 ? try(compute_config.value.node_role_arn, aws_iam_role.eks_auto[0].arn, null) : null + enabled = compute_config.value.enabled + node_pools = compute_config.value.node_pools + node_role_arn = compute_config.value.node_pools != null ? try(compute_config.value.node_role_arn, aws_iam_role.eks_auto[0].arn, null) : null } } vpc_config { - security_group_ids = compact(distinct(concat(var.cluster_additional_security_group_ids, [local.cluster_security_group_id]))) + security_group_ids = compact(distinct(concat(var.additional_security_group_ids, [local.security_group_id]))) subnet_ids = coalescelist(var.control_plane_subnet_ids, var.subnet_ids) - endpoint_private_access = var.cluster_endpoint_private_access - endpoint_public_access = var.cluster_endpoint_public_access - public_access_cidrs = var.cluster_endpoint_public_access_cidrs + endpoint_private_access = var.endpoint_private_access + endpoint_public_access = var.endpoint_public_access + public_access_cidrs = var.endpoint_public_access_cidrs } dynamic "kubernetes_network_config" { @@ -84,9 +87,9 @@ resource "aws_eks_cluster" "this" { } } - ip_family = var.cluster_ip_family - service_ipv4_cidr = var.cluster_service_ipv4_cidr - service_ipv6_cidr = var.cluster_service_ipv6_cidr + ip_family = var.ip_family + service_ipv4_cidr = var.service_ipv4_cidr + service_ipv6_cidr = var.service_ipv6_cidr } } @@ -95,13 +98,22 @@ resource "aws_eks_cluster" "this" { content { control_plane_instance_type = outpost_config.value.control_plane_instance_type - outpost_arns = outpost_config.value.outpost_arns + + dynamic "control_plane_placement" { + for_each = outpost_config.value.control_plane_placement != null ? [outpost_config.value.control_plane_placement] : [] + + content { + group_name = control_plane_placement.value.group_name + } + } + + outpost_arns = outpost_config.value.outpost_arns } } dynamic "encryption_config" { # Not available on Outposts - for_each = local.enable_cluster_encryption_config ? [var.cluster_encryption_config] : [] + for_each = local.enable_encryption_config ? [var.encryption_config] : [] content { provider { @@ -113,7 +125,7 @@ resource "aws_eks_cluster" "this" { dynamic "remote_network_config" { # Not valid on Outposts - for_each = length(var.cluster_remote_network_config) > 0 && !local.create_outposts_local_cluster ? [var.cluster_remote_network_config] : [] + for_each = var.remote_network_config != null && !local.create_outposts_local_cluster ? [var.remote_network_config] : [] content { dynamic "remote_node_networks" { @@ -125,7 +137,7 @@ resource "aws_eks_cluster" "this" { } dynamic "remote_pod_networks" { - for_each = try([remote_network_config.value.remote_pod_networks], []) + for_each = remote_network_config.value.remote_pod_networks != null ? [remote_network_config.value.remote_pod_networks] : [] content { cidrs = remote_pod_networks.value.cidrs @@ -145,18 +157,18 @@ resource "aws_eks_cluster" "this" { } dynamic "upgrade_policy" { - for_each = length(var.cluster_upgrade_policy) > 0 ? [var.cluster_upgrade_policy] : [] + for_each = var.upgrade_policy != null ? [var.upgrade_policy] : [] content { - support_type = try(upgrade_policy.value.support_type, null) + support_type = upgrade_policy.value.support_type } } dynamic "zonal_shift_config" { - for_each = length(var.cluster_zonal_shift_config) > 0 ? [var.cluster_zonal_shift_config] : [] + for_each = var.zonal_shift_config != null ? [var.zonal_shift_config] : [] content { - enabled = try(zonal_shift_config.value.enabled, null) + enabled = zonal_shift_config.value.enabled } } @@ -166,10 +178,14 @@ resource "aws_eks_cluster" "this" { var.cluster_tags, ) - timeouts { - create = try(var.cluster_timeouts.create, null) - update = try(var.cluster_timeouts.update, null) - delete = try(var.cluster_timeouts.delete, null) + dynamic "timeouts" { + for_each = var.timeouts != null ? [var.timeouts] : [] + + content { + create = var.timeouts.create + update = var.timeouts.update + delete = var.timeouts.delete + } } depends_on = [ @@ -182,7 +198,8 @@ resource "aws_eks_cluster" "this" { lifecycle { ignore_changes = [ - access_config[0].bootstrap_cluster_creator_admin_permissions + access_config[0].bootstrap_cluster_creator_admin_permissions, + bootstrap_self_managed_addons, ] } } @@ -192,9 +209,11 @@ resource "aws_ec2_tag" "cluster_primary_security_group" { # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2006 # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2008 for_each = { for k, v in merge(var.tags, var.cluster_tags) : - k => v if local.create && k != "Name" && var.create_cluster_primary_security_group_tags + k => v if local.create && k != "Name" && var.create_primary_security_group_tags } + region = var.region + resource_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id key = each.key value = each.value @@ -203,7 +222,9 @@ resource "aws_ec2_tag" "cluster_primary_security_group" { resource "aws_cloudwatch_log_group" "this" { count = local.create && var.create_cloudwatch_log_group ? 1 : 0 - name = "/aws/eks/${var.cluster_name}/cluster" + region = var.region + + name = "/aws/eks/${var.name}/cluster" retention_in_days = var.cloudwatch_log_group_retention_in_days kms_key_id = var.cloudwatch_log_group_kms_key_id log_group_class = var.cloudwatch_log_group_class @@ -211,7 +232,7 @@ resource "aws_cloudwatch_log_group" "this" { tags = merge( var.tags, var.cloudwatch_log_group_tags, - { Name = "/aws/eks/${var.cluster_name}/cluster" } + { Name = "/aws/eks/${var.name}/cluster" } ) } @@ -222,7 +243,7 @@ resource "aws_cloudwatch_log_group" "this" { locals { # This replaces the one time logic from the EKS API with something that can be # better controlled by users through Terraform - bootstrap_cluster_creator_admin_permissions = { + bootstrap_cluster_creator_admin_permissions = { for k, v in { cluster_creator = { principal_arn = try(data.aws_iam_session_context.current[0].issuer_arn, "") type = "STANDARD" @@ -236,11 +257,11 @@ locals { } } } - } + } : k => v if var.enable_cluster_creator_admin_permissions } # Merge the bootstrap behavior with the entries that users provide merged_access_entries = merge( - { for k, v in local.bootstrap_cluster_creator_admin_permissions : k => v if var.enable_cluster_creator_admin_permissions }, + local.bootstrap_cluster_creator_admin_permissions, var.access_entries, ) @@ -248,7 +269,7 @@ locals { # associations within a single entry flattened_access_entries = flatten([ for entry_key, entry_val in local.merged_access_entries : [ - for pol_key, pol_val in lookup(entry_val, "policy_associations", {}) : + for pol_key, pol_val in try(entry_val.policy_associations, {}) : merge( { principal_arn = entry_val.principal_arn @@ -258,7 +279,7 @@ locals { { for k, v in { association_policy_arn = pol_val.policy_arn association_access_scope_type = pol_val.access_scope.type - association_access_scope_namespaces = lookup(pol_val.access_scope, "namespaces", []) + association_access_scope_namespaces = try(pol_val.access_scope.namespaces, null) } : k => v if !contains(["EC2_LINUX", "EC2_WINDOWS", "FARGATE_LINUX", "HYBRID_LINUX"], lookup(entry_val, "type", "STANDARD")) }, ) ] @@ -268,20 +289,27 @@ locals { resource "aws_eks_access_entry" "this" { for_each = { for k, v in local.merged_access_entries : k => v if local.create } + region = var.region + cluster_name = aws_eks_cluster.this[0].id kubernetes_groups = try(each.value.kubernetes_groups, null) principal_arn = each.value.principal_arn - type = try(each.value.type, "STANDARD") + type = try(each.value.type, null) user_name = try(each.value.user_name, null) - tags = merge(var.tags, try(each.value.tags, {})) + tags = merge( + var.tags, + try(each.value.tags, {}), + ) } resource "aws_eks_access_policy_association" "this" { for_each = { for k, v in local.flattened_access_entries : "${v.entry_key}_${v.pol_key}" => v if local.create } + region = var.region + access_scope { - namespaces = try(each.value.association_access_scope_namespaces, []) + namespaces = each.value.association_access_scope_namespaces type = each.value.association_access_scope_type } @@ -301,11 +329,13 @@ resource "aws_eks_access_policy_association" "this" { module "kms" { source = "terraform-aws-modules/kms/aws" - version = "2.1.0" # Note - be mindful of Terraform/provider version compatibility between modules + version = "4.0.0" # Note - be mindful of Terraform/provider version compatibility between modules + + create = local.create && var.create_kms_key && local.enable_encryption_config # not valid on Outposts - create = local.create && var.create_kms_key && local.enable_cluster_encryption_config # not valid on Outposts + region = var.region - description = coalesce(var.kms_key_description, "${var.cluster_name} cluster encryption key") + description = coalesce(var.kms_key_description, "${var.name} cluster encryption key") key_usage = "ENCRYPT_DECRYPT" deletion_window_in_days = var.kms_key_deletion_window_in_days enable_key_rotation = var.enable_kms_key_rotation @@ -314,7 +344,7 @@ module "kms" { enable_default_policy = var.kms_key_enable_default_policy key_owners = var.kms_key_owners key_administrators = coalescelist(var.kms_key_administrators, [try(data.aws_iam_session_context.current[0].issuer_arn, "")]) - key_users = concat([local.cluster_role], var.kms_key_users) + key_users = concat([local.role_arn], var.kms_key_users) key_service_users = var.kms_key_service_users source_policy_documents = var.kms_key_source_policy_documents override_policy_documents = var.kms_key_override_policy_documents @@ -323,7 +353,7 @@ module "kms" { aliases = var.kms_key_aliases computed_aliases = { # Computed since users can pass in computed values for cluster name such as random provider resources - cluster = { name = "eks/${var.cluster_name}" } + cluster = { name = "eks/${var.name}" } } tags = merge( @@ -338,10 +368,10 @@ module "kms" { ################################################################################ locals { - cluster_sg_name = coalesce(var.cluster_security_group_name, "${var.cluster_name}-cluster") - create_cluster_sg = local.create && var.create_cluster_security_group + security_group_name = coalesce(var.security_group_name, "${var.name}-cluster") + create_security_group = local.create && var.create_security_group - cluster_security_group_id = local.create_cluster_sg ? aws_security_group.cluster[0].id : var.cluster_security_group_id + security_group_id = local.create_security_group ? aws_security_group.cluster[0].id : var.security_group_id # Do not add rules to node security group if the module is not creating it cluster_security_group_rules = { for k, v in { @@ -357,17 +387,19 @@ locals { } resource "aws_security_group" "cluster" { - count = local.create_cluster_sg ? 1 : 0 + count = local.create_security_group ? 1 : 0 - name = var.cluster_security_group_use_name_prefix ? null : local.cluster_sg_name - name_prefix = var.cluster_security_group_use_name_prefix ? "${local.cluster_sg_name}${var.prefix_separator}" : null - description = var.cluster_security_group_description + region = var.region + + name = var.security_group_use_name_prefix ? null : local.security_group_name + name_prefix = var.security_group_use_name_prefix ? "${local.security_group_name}${var.prefix_separator}" : null + description = var.security_group_description vpc_id = var.vpc_id tags = merge( var.tags, - { "Name" = local.cluster_sg_name }, - var.cluster_security_group_tags + { "Name" = local.security_group_name }, + var.security_group_tags ) lifecycle { @@ -378,23 +410,22 @@ resource "aws_security_group" "cluster" { resource "aws_security_group_rule" "cluster" { for_each = { for k, v in merge( local.cluster_security_group_rules, - var.cluster_security_group_additional_rules - ) : k => v if local.create_cluster_sg } + var.security_group_additional_rules + ) : k => v if local.create_security_group } - # Required - security_group_id = aws_security_group.cluster[0].id - protocol = each.value.protocol - from_port = each.value.from_port - to_port = each.value.to_port - type = each.value.type + region = var.region - # Optional - description = lookup(each.value, "description", null) - cidr_blocks = lookup(each.value, "cidr_blocks", null) - ipv6_cidr_blocks = lookup(each.value, "ipv6_cidr_blocks", null) - prefix_list_ids = lookup(each.value, "prefix_list_ids", null) - self = lookup(each.value, "self", null) - source_security_group_id = try(each.value.source_node_security_group, false) ? local.node_security_group_id : lookup(each.value, "source_security_group_id", null) + security_group_id = aws_security_group.cluster[0].id + protocol = each.value.protocol + from_port = each.value.from_port + to_port = each.value.to_port + type = each.value.type + description = try(each.value.description, null) + cidr_blocks = try(each.value.cidr_blocks, null) + ipv6_cidr_blocks = try(each.value.ipv6_cidr_blocks, null) + prefix_list_ids = try(each.value.prefix_list_ids, null) + self = try(each.value.self, null) + source_security_group_id = try(each.value.source_node_security_group, false) ? local.node_security_group_id : try(each.value.source_security_group_id, null) } ################################################################################ @@ -413,7 +444,7 @@ data "tls_certificate" "this" { # Not available on outposts count = local.create_oidc_provider && var.include_oidc_root_ca_thumbprint ? 1 : 0 - url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer + url = local.dualstack_oidc_issuer_url } resource "aws_iam_openid_connect_provider" "oidc_provider" { @@ -422,10 +453,10 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { client_id_list = distinct(compact(concat(["sts.amazonaws.com"], var.openid_connect_audiences))) thumbprint_list = concat(local.oidc_root_ca_thumbprint, var.custom_oidc_thumbprints) - url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer + url = local.dualstack_oidc_issuer_url tags = merge( - { Name = "${var.cluster_name}-eks-irsa" }, + { Name = "${var.name}-eks-irsa" }, var.tags ) } @@ -436,10 +467,10 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { locals { create_iam_role = local.create && var.create_iam_role - iam_role_name = coalesce(var.iam_role_name, "${var.cluster_name}-cluster") + iam_role_name = coalesce(var.iam_role_name, "${var.name}-cluster") iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" - cluster_encryption_policy_name = coalesce(var.cluster_encryption_policy_name, "${local.iam_role_name}-ClusterEncryption") + cluster_encryption_policy_name = coalesce(var.encryption_policy_name, "${local.iam_role_name}-ClusterEncryption") # Standard EKS cluster eks_standard_iam_role_policies = { for k, v in { @@ -459,11 +490,6 @@ locals { eks_outpost_iam_role_policies = { for k, v in { AmazonEKSClusterPolicy = "${local.iam_role_policy_prefix}/AmazonEKSLocalOutpostClusterPolicy" } : k => v if local.create_outposts_local_cluster && !local.auto_mode_enabled } - - # Security groups for pods - eks_sgpp_iam_role_policies = { for k, v in { - AmazonEKSVPCResourceController = "${local.iam_role_policy_prefix}/AmazonEKSVPCResourceController" - } : k => v if var.enable_security_groups_for_pods && !local.create_outposts_local_cluster && !local.auto_mode_enabled } } data "aws_iam_policy_document" "assume_role_policy" { @@ -513,7 +539,6 @@ resource "aws_iam_role_policy_attachment" "this" { local.eks_standard_iam_role_policies, local.eks_auto_mode_iam_role_policies, local.eks_outpost_iam_role_policies, - local.eks_sgpp_iam_role_policies, ) : k => v if local.create_iam_role } policy_arn = each.value @@ -530,7 +555,7 @@ resource "aws_iam_role_policy_attachment" "additional" { # Using separate attachment due to `The "for_each" value depends on resource attributes that cannot be determined until apply` resource "aws_iam_role_policy_attachment" "cluster_encryption" { # Encryption config not available on Outposts - count = local.create_iam_role && var.attach_cluster_encryption_policy && local.enable_cluster_encryption_config ? 1 : 0 + count = local.create_iam_role && var.attach_encryption_policy && local.enable_encryption_config ? 1 : 0 policy_arn = aws_iam_policy.cluster_encryption[0].arn role = aws_iam_role.this[0].name @@ -538,12 +563,12 @@ resource "aws_iam_role_policy_attachment" "cluster_encryption" { resource "aws_iam_policy" "cluster_encryption" { # Encryption config not available on Outposts - count = local.create_iam_role && var.attach_cluster_encryption_policy && local.enable_cluster_encryption_config ? 1 : 0 + count = local.create_iam_role && var.attach_encryption_policy && local.enable_encryption_config ? 1 : 0 - name = var.cluster_encryption_policy_use_name_prefix ? null : local.cluster_encryption_policy_name - name_prefix = var.cluster_encryption_policy_use_name_prefix ? local.cluster_encryption_policy_name : null - description = var.cluster_encryption_policy_description - path = var.cluster_encryption_policy_path + name = var.encryption_policy_use_name_prefix ? null : local.cluster_encryption_policy_name + name_prefix = var.encryption_policy_use_name_prefix ? local.cluster_encryption_policy_name : null + description = var.encryption_policy_description + path = var.encryption_policy_path policy = jsonencode({ Version = "2012-10-17" @@ -556,12 +581,12 @@ resource "aws_iam_policy" "cluster_encryption" { "kms:DescribeKey", ] Effect = "Allow" - Resource = var.create_kms_key ? module.kms.key_arn : var.cluster_encryption_config.provider_key_arn + Resource = var.create_kms_key ? module.kms.key_arn : var.encryption_config.provider_key_arn }, ] }) - tags = merge(var.tags, var.cluster_encryption_policy_tags) + tags = merge(var.tags, var.encryption_policy_tags) } data "aws_iam_policy_document" "custom" { @@ -722,32 +747,30 @@ resource "aws_iam_role_policy_attachment" "custom" { # EKS Addons ################################################################################ -locals { - # TODO - Set to `NONE` on next breaking change when default addons are disabled - resolve_conflicts_on_create_default = coalesce(var.bootstrap_self_managed_addons, true) ? "OVERWRITE" : "NONE" -} - data "aws_eks_addon_version" "this" { - for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } + for_each = var.addons != null && local.create && !local.create_outposts_local_cluster ? var.addons : {} - addon_name = try(each.value.name, each.key) - kubernetes_version = coalesce(var.cluster_version, aws_eks_cluster.this[0].version) - # TODO - Set default fallback to `true` on next breaking change - most_recent = try(each.value.most_recent, null) + region = var.region + + addon_name = coalesce(each.value.name, each.key) + kubernetes_version = coalesce(var.kubernetes_version, aws_eks_cluster.this[0].version) + most_recent = each.value.most_recent } resource "aws_eks_addon" "this" { # Not supported on outposts - for_each = { for k, v in var.cluster_addons : k => v if !try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } + for_each = var.addons != null && local.create && !local.create_outposts_local_cluster ? { for k, v in var.addons : k => v if !v.before_compute } : {} + + region = var.region cluster_name = aws_eks_cluster.this[0].id - addon_name = try(each.value.name, each.key) + addon_name = coalesce(each.value.name, each.key) - addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) - configuration_values = try(each.value.configuration_values, null) + addon_version = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) + configuration_values = each.value.configuration_values dynamic "pod_identity_association" { - for_each = try(each.value.pod_identity_association, []) + for_each = each.value.pod_identity_association != null ? each.value.pod_identity_association : [] content { role_arn = pod_identity_association.value.role_arn @@ -755,39 +778,44 @@ resource "aws_eks_addon" "this" { } } - preserve = try(each.value.preserve, true) - # TODO - Set to `NONE` on next breaking change when default addons are disabled - resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, local.resolve_conflicts_on_create_default) - resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE") - service_account_role_arn = try(each.value.service_account_role_arn, null) + preserve = each.value.preserve + resolve_conflicts_on_create = each.value.resolve_conflicts_on_create + resolve_conflicts_on_update = each.value.resolve_conflicts_on_update + service_account_role_arn = each.value.service_account_role_arn timeouts { - create = try(each.value.timeouts.create, var.cluster_addons_timeouts.create, null) - update = try(each.value.timeouts.update, var.cluster_addons_timeouts.update, null) - delete = try(each.value.timeouts.delete, var.cluster_addons_timeouts.delete, null) + create = try(each.value.timeouts.create, var.addons_timeouts.create, null) + update = try(each.value.timeouts.update, var.addons_timeouts.update, null) + delete = try(each.value.timeouts.delete, var.addons_timeouts.delete, null) } + tags = merge( + var.tags, + each.value.tags, + ) + + # before_compute = false depends_on = [ module.fargate_profile, module.eks_managed_node_group, module.self_managed_node_group, ] - - tags = merge(var.tags, try(each.value.tags, {})) } resource "aws_eks_addon" "before_compute" { # Not supported on outposts - for_each = { for k, v in var.cluster_addons : k => v if try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } + for_each = var.addons != null && local.create && !local.create_outposts_local_cluster ? { for k, v in var.addons : k => v if v.before_compute } : {} + + region = var.region cluster_name = aws_eks_cluster.this[0].id - addon_name = try(each.value.name, each.key) + addon_name = coalesce(each.value.name, each.key) - addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) - configuration_values = try(each.value.configuration_values, null) + addon_version = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) + configuration_values = each.value.configuration_values dynamic "pod_identity_association" { - for_each = try(each.value.pod_identity_association, []) + for_each = each.value.pod_identity_association != null ? each.value.pod_identity_association : [] content { role_arn = pod_identity_association.value.role_arn @@ -795,19 +823,21 @@ resource "aws_eks_addon" "before_compute" { } } - preserve = try(each.value.preserve, true) - # TODO - Set to `NONE` on next breaking change when default addons are disabled - resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, local.resolve_conflicts_on_create_default) - resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE") - service_account_role_arn = try(each.value.service_account_role_arn, null) + preserve = each.value.preserve + resolve_conflicts_on_create = each.value.resolve_conflicts_on_create + resolve_conflicts_on_update = each.value.resolve_conflicts_on_update + service_account_role_arn = each.value.service_account_role_arn timeouts { - create = try(each.value.timeouts.create, var.cluster_addons_timeouts.create, null) - update = try(each.value.timeouts.update, var.cluster_addons_timeouts.update, null) - delete = try(each.value.timeouts.delete, var.cluster_addons_timeouts.delete, null) + create = try(each.value.timeouts.create, var.addons_timeouts.create, null) + update = try(each.value.timeouts.update, var.addons_timeouts.update, null) + delete = try(each.value.timeouts.delete, var.addons_timeouts.delete, null) } - tags = merge(var.tags, try(each.value.tags, {})) + tags = merge( + var.tags, + each.value.tags, + ) } ################################################################################ @@ -815,33 +845,28 @@ resource "aws_eks_addon" "before_compute" { # Note - this is different from IRSA ################################################################################ -locals { - # Maintain current behavior for <= 1.29, remove default for >= 1.30 - # `null` will return the latest Kubernetes version from the EKS API, which at time of writing is 1.30 - # https://github.com/kubernetes/kubernetes/pull/123561 - # TODO - remove on next breaking change in conjunction with issuer URL change below - idpc_backwards_compat_version = contains(["1.21", "1.22", "1.23", "1.24", "1.25", "1.26", "1.27", "1.28", "1.29"], coalesce(var.cluster_version, "1.30")) - idpc_issuer_url = local.idpc_backwards_compat_version ? try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, null) : null -} - resource "aws_eks_identity_provider_config" "this" { - for_each = { for k, v in var.cluster_identity_providers : k => v if local.create && !local.create_outposts_local_cluster } + for_each = var.identity_providers != null && local.create && !local.create_outposts_local_cluster ? var.identity_providers : {} + + region = var.region cluster_name = aws_eks_cluster.this[0].id oidc { client_id = each.value.client_id - groups_claim = lookup(each.value, "groups_claim", null) - groups_prefix = lookup(each.value, "groups_prefix", null) + groups_claim = each.value.groups_claim + groups_prefix = each.value.groups_prefix identity_provider_config_name = try(each.value.identity_provider_config_name, each.key) - # TODO - make argument explicitly required on next breaking change - issuer_url = try(each.value.issuer_url, local.idpc_issuer_url) - required_claims = lookup(each.value, "required_claims", null) - username_claim = lookup(each.value, "username_claim", null) - username_prefix = lookup(each.value, "username_prefix", null) + issuer_url = each.value.issuer_url + required_claims = each.value.required_claims + username_claim = each.value.username_claim + username_prefix = each.value.username_prefix } - tags = merge(var.tags, try(each.value.tags, {})) + tags = merge( + var.tags, + each.value.tags, + ) } ################################################################################ @@ -850,7 +875,7 @@ resource "aws_eks_identity_provider_config" "this" { locals { create_node_iam_role = local.create && var.create_node_iam_role && local.auto_mode_enabled - node_iam_role_name = coalesce(var.node_iam_role_name, "${var.cluster_name}-eks-auto") + node_iam_role_name = coalesce(var.node_iam_role_name, "${var.name}-eks-auto") } data "aws_iam_policy_document" "node_assume_role_policy" { diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 185335b16a..610051bc9c 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -9,7 +9,7 @@ See [`tests/user-data/`](https://github.com/terraform-aws-modules/terraform-aws- | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | | [cloudinit](#requirement\_cloudinit) | >= 2.0 | | [null](#requirement\_null) | >= 3.0 | @@ -30,14 +30,14 @@ No modules. |------|------| | [null_resource.validate_cluster_service_cidr](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [cloudinit_config.al2023_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | -| [cloudinit_config.linux_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | +| [cloudinit_config.al2_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [additional\_cluster\_dns\_ips](#input\_additional\_cluster\_dns\_ips) | Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*` | `list(string)` | `[]` | no | -| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `null` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `"AL2023_x86_64_STANDARD"` | no | | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | | [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | | [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | @@ -46,11 +46,9 @@ No modules. | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | | [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | -| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | [Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [create](#input\_create) | Determines whether to create user-data or not | `bool` | `true` | no | | [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template | `bool` | `false` | no | | [is\_eks\_managed\_node\_group](#input\_is\_eks\_managed\_node\_group) | Determines whether the user data is used on nodes in an EKS managed node group. Used to determine if user data will be appended or not | `bool` | `true` | no | -| [platform](#input\_platform) | [DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | `string` | `"linux"` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | | [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | | [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | @@ -59,6 +57,5 @@ No modules. | Name | Description | |------|-------------| -| [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023, or `windows | | [user\_data](#output\_user\_data) | Base64 encoded user data rendered for the provided inputs | diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index c24074af15..c394421c4c 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -7,51 +7,47 @@ resource "null_resource" "validate_cluster_service_cidr" { precondition { # The length 6 is currently arbitrary, but it's a safe bet that the CIDR will be longer than that # The main point is that a value needs to be provided when `create = true` - condition = var.create ? length(local.cluster_service_cidr) > 6 : true + condition = var.create ? length(var.cluster_service_cidr) > 6 : true error_message = "`cluster_service_cidr` is required when `create = true`." } } } locals { - # Converts AMI type into user data type that represents the underlying format (bash, toml, PS1, nodeadm) - # TODO - platform will be removed in v21.0 and only `ami_type` will be valid - ami_type_to_user_data_type = { - AL2_x86_64 = "linux" - AL2_x86_64_GPU = "linux" - AL2_ARM_64 = "linux" - BOTTLEROCKET_ARM_64 = "bottlerocket" - BOTTLEROCKET_x86_64 = "bottlerocket" - BOTTLEROCKET_ARM_64_FIPS = "bottlerocket" - BOTTLEROCKET_x86_64_FIPS = "bottlerocket" - BOTTLEROCKET_ARM_64_NVIDIA = "bottlerocket" - BOTTLEROCKET_x86_64_NVIDIA = "bottlerocket" - WINDOWS_CORE_2019_x86_64 = "windows" - WINDOWS_FULL_2019_x86_64 = "windows" - WINDOWS_CORE_2022_x86_64 = "windows" - WINDOWS_FULL_2022_x86_64 = "windows" - AL2023_x86_64_STANDARD = "al2023" - AL2023_ARM_64_STANDARD = "al2023" - AL2023_x86_64_NEURON = "al2023" - AL2023_x86_64_NVIDIA = "al2023" - AL2023_ARM_64_NVIDIA = "al2023" - } - # Try to use `ami_type` first, but fall back to current, default behavior - # TODO - will be removed in v21.0 - user_data_type = try(local.ami_type_to_user_data_type[var.ami_type], var.platform) - - template_path = { - al2023 = "${path.module}/../../templates/al2023_user_data.tpl" - bottlerocket = "${path.module}/../../templates/bottlerocket_user_data.tpl" - linux = "${path.module}/../../templates/linux_user_data.tpl" - windows = "${path.module}/../../templates/windows_user_data.tpl" + is_al2 = startswith(var.ami_type, "AL2_") + is_al2023 = startswith(var.ami_type, "AL2023_") + + # Converts AMI type into user data template path + ami_type_to_user_data_path = { + AL2_ARM_64 = "${path.module}/../../templates/al2_user_data.tpl" + AL2_x86_64 = "${path.module}/../../templates/al2_user_data.tpl" + AL2_x86_64_GPU = "${path.module}/../../templates/al2_user_data.tpl" + + AL2023_x86_64_STANDARD = "${path.module}/../../templates/al2023_user_data.tpl" + AL2023_ARM_64_STANDARD = "${path.module}/../../templates/al2023_user_data.tpl" + AL2023_x86_64_NEURON = "${path.module}/../../templates/al2023_user_data.tpl" + AL2023_x86_64_NVIDIA = "${path.module}/../../templates/al2023_user_data.tpl" + AL2023_ARM_64_NVIDIA = "${path.module}/../../templates/al2023_user_data.tpl" + + BOTTLEROCKET_ARM_64 = "${path.module}/../../templates/bottlerocket_user_data.tpl" + BOTTLEROCKET_x86_64 = "${path.module}/../../templates/bottlerocket_user_data.tpl" + BOTTLEROCKET_ARM_64_FIPS = "${path.module}/../../templates/bottlerocket_user_data.tpl" + BOTTLEROCKET_x86_64_FIPS = "${path.module}/../../templates/bottlerocket_user_data.tpl" + BOTTLEROCKET_ARM_64_NVIDIA = "${path.module}/../../templates/bottlerocket_user_data.tpl" + BOTTLEROCKET_x86_64_NVIDIA = "${path.module}/../../templates/bottlerocket_user_data.tpl" + + WINDOWS_CORE_2019_x86_64 = "${path.module}/../../templates/windows_user_data.tpl" + WINDOWS_FULL_2019_x86_64 = "${path.module}/../../templates/windows_user_data.tpl" + WINDOWS_CORE_2022_x86_64 = "${path.module}/../../templates/windows_user_data.tpl" + WINDOWS_FULL_2022_x86_64 = "${path.module}/../../templates/windows_user_data.tpl" + + CUSTOM = var.user_data_template_path } + user_data_path = coalesce(var.user_data_template_path, local.ami_type_to_user_data_path[var.ami_type]) - cluster_service_cidr = try(coalesce(var.cluster_service_ipv4_cidr, var.cluster_service_cidr), "") - cluster_dns_ips = flatten(concat([try(cidrhost(local.cluster_service_cidr, 10), "")], var.additional_cluster_dns_ips)) + cluster_dns_ips = flatten(concat([try(cidrhost(var.cluster_service_cidr, 10), "")], var.additional_cluster_dns_ips)) - user_data = base64encode(templatefile( - coalesce(var.user_data_template_path, local.template_path[local.user_data_type]), + user_data = var.create ? base64encode(templatefile(local.user_data_path, { # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami enable_bootstrap_user_data = var.enable_bootstrap_user_data @@ -61,7 +57,7 @@ locals { cluster_endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 - cluster_service_cidr = local.cluster_service_cidr + cluster_service_cidr = var.cluster_service_cidr cluster_ip_family = var.cluster_ip_family # Bottlerocket @@ -72,22 +68,13 @@ locals { pre_bootstrap_user_data = var.pre_bootstrap_user_data post_bootstrap_user_data = var.post_bootstrap_user_data } - )) + )) : "" - user_data_type_to_rendered = { - al2023 = { - user_data = var.create ? try(data.cloudinit_config.al2023_eks_managed_node_group[0].rendered, local.user_data) : "" - } - bottlerocket = { - user_data = var.create && local.user_data_type == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? local.user_data : "" - } - linux = { - user_data = var.create ? try(data.cloudinit_config.linux_eks_managed_node_group[0].rendered, local.user_data) : "" - } - windows = { - user_data = var.create && local.user_data_type == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? local.user_data : "" - } - } + user_data_type_to_rendered = try(coalesce( + local.is_al2 ? try(data.cloudinit_config.al2_eks_managed_node_group[0].rendered, local.user_data) : null, + local.is_al2023 ? try(data.cloudinit_config.al2023_eks_managed_node_group[0].rendered, local.user_data) : null, + local.user_data, + ), "") } # https://github.com/aws/containers-roadmap/issues/596#issuecomment-675097667 @@ -97,8 +84,8 @@ locals { # this merging will NOT happen and you are responsible for nodes joining the cluster. # See docs for more details -> https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data -data "cloudinit_config" "linux_eks_managed_node_group" { - count = var.create && local.user_data_type == "linux" && var.is_eks_managed_node_group && !var.enable_bootstrap_user_data && var.pre_bootstrap_user_data != "" && var.user_data_template_path == "" ? 1 : 0 +data "cloudinit_config" "al2_eks_managed_node_group" { + count = var.create && local.is_al2 && var.is_eks_managed_node_group && !var.enable_bootstrap_user_data && var.pre_bootstrap_user_data != "" && var.user_data_template_path == "" ? 1 : 0 base64_encode = true gzip = false @@ -129,7 +116,7 @@ locals { } data "cloudinit_config" "al2023_eks_managed_node_group" { - count = var.create && local.user_data_type == "al2023" && length(local.nodeadm_cloudinit) > 0 ? 1 : 0 + count = var.create && local.is_al2023 && length(local.nodeadm_cloudinit) > 0 ? 1 : 0 base64_encode = true gzip = false diff --git a/modules/_user_data/outputs.tf b/modules/_user_data/outputs.tf index 7bebb3f218..dda4b5195d 100644 --- a/modules/_user_data/outputs.tf +++ b/modules/_user_data/outputs.tf @@ -1,9 +1,4 @@ output "user_data" { description = "Base64 encoded user data rendered for the provided inputs" - value = try(local.user_data_type_to_rendered[local.user_data_type].user_data, null) -} - -output "platform" { - description = "[DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023, or `windows`" - value = local.user_data_type + value = local.user_data_type_to_rendered } diff --git a/modules/_user_data/variables.tf b/modules/_user_data/variables.tf index d5a1ef1bd3..bfc32ab688 100644 --- a/modules/_user_data/variables.tf +++ b/modules/_user_data/variables.tf @@ -2,97 +2,98 @@ variable "create" { description = "Determines whether to create user-data or not" type = bool default = true -} - -variable "platform" { - description = "[DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" - type = string - default = "linux" + nullable = false } variable "ami_type" { description = "Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values" type = string - default = null + default = "AL2023_x86_64_STANDARD" + nullable = false } variable "enable_bootstrap_user_data" { description = "Determines whether the bootstrap configurations are populated within the user data template" type = bool default = false + nullable = false } variable "is_eks_managed_node_group" { description = "Determines whether the user data is used on nodes in an EKS managed node group. Used to determine if user data will be appended or not" type = bool default = true + nullable = false } variable "cluster_name" { description = "Name of the EKS cluster" type = string default = "" + nullable = false } variable "cluster_endpoint" { description = "Endpoint of associated EKS cluster" type = string default = "" + nullable = false } variable "cluster_auth_base64" { description = "Base64 encoded CA of associated EKS cluster" type = string default = "" + nullable = false } variable "cluster_service_cidr" { description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" type = string default = "" + nullable = false } variable "cluster_ip_family" { description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" type = string default = "ipv4" + nullable = false } variable "additional_cluster_dns_ips" { description = "Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*`" type = list(string) default = [] -} - -# TODO - remove at next breaking change -variable "cluster_service_ipv4_cidr" { - description = "[Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" - type = string - default = null + nullable = false } variable "pre_bootstrap_user_data" { description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string default = "" + nullable = false } variable "post_bootstrap_user_data" { description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string default = "" + nullable = false } variable "bootstrap_extra_args" { description = "Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" type = string default = "" + nullable = false } variable "user_data_template_path" { description = "Path to a local, custom user data template file to use when rendering user data" type = string default = "" + nullable = false } variable "cloudinit_pre_nodeadm" { @@ -103,7 +104,8 @@ variable "cloudinit_pre_nodeadm" { filename = optional(string) merge_type = optional(string) })) - default = [] + default = [] + nullable = false } variable "cloudinit_post_nodeadm" { @@ -114,5 +116,6 @@ variable "cloudinit_post_nodeadm" { filename = optional(string) merge_type = optional(string) })) - default = [] + default = [] + nullable = false } diff --git a/modules/_user_data/versions.tf b/modules/_user_data/versions.tf index 9219addeda..a9802b0dea 100644 --- a/modules/_user_data/versions.tf +++ b/modules/_user_data/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { cloudinit = { diff --git a/modules/aws-auth/README.md b/modules/aws-auth/README.md deleted file mode 100644 index d4b97f14ca..0000000000 --- a/modules/aws-auth/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# `aws-auth` Module - -Configuration in this directory creates/updates the `aws-auth` ConfigMap. - -```hcl -module "eks" { - source = "terraform-aws-modules/eks/aws//modules/aws-auth" - version = "~> 20.0" - - manage_aws_auth_configmap = true - - aws_auth_roles = [ - { - rolearn = "arn:aws:iam::66666666666:role/role1" - username = "role1" - groups = ["system:masters"] - }, - ] - - aws_auth_users = [ - { - userarn = "arn:aws:iam::66666666666:user/user1" - username = "user1" - groups = ["system:masters"] - }, - { - userarn = "arn:aws:iam::66666666666:user/user2" - username = "user2" - groups = ["system:masters"] - }, - ] - - aws_auth_accounts = [ - "777777777777", - "888888888888", - ] -} -``` - -## Usage - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [kubernetes](#requirement\_kubernetes) | >= 2.20 | - -## Providers - -| Name | Version | -|------|---------| -| [kubernetes](#provider\_kubernetes) | >= 2.20 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | -| [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [aws\_auth\_accounts](#input\_aws\_auth\_accounts) | List of account maps to add to the aws-auth configmap | `list(any)` | `[]` | no | -| [aws\_auth\_roles](#input\_aws\_auth\_roles) | List of role maps to add to the aws-auth configmap | `list(any)` | `[]` | no | -| [aws\_auth\_users](#input\_aws\_auth\_users) | List of user maps to add to the aws-auth configmap | `list(any)` | `[]` | no | -| [create](#input\_create) | Controls if resources should be created (affects all resources) | `bool` | `true` | no | -| [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | -| [manage\_aws\_auth\_configmap](#input\_manage\_aws\_auth\_configmap) | Determines whether to manage the aws-auth configmap | `bool` | `true` | no | - -## Outputs - -No outputs. - diff --git a/modules/aws-auth/main.tf b/modules/aws-auth/main.tf deleted file mode 100644 index 2f7e9694a7..0000000000 --- a/modules/aws-auth/main.tf +++ /dev/null @@ -1,47 +0,0 @@ - -################################################################################ -# aws-auth configmap -################################################################################ - -locals { - aws_auth_configmap_data = { - mapRoles = yamlencode(var.aws_auth_roles) - mapUsers = yamlencode(var.aws_auth_users) - mapAccounts = yamlencode(var.aws_auth_accounts) - } -} - -resource "kubernetes_config_map" "aws_auth" { - count = var.create && var.create_aws_auth_configmap ? 1 : 0 - - metadata { - name = "aws-auth" - namespace = "kube-system" - } - - data = local.aws_auth_configmap_data - - lifecycle { - # We are ignoring the data here since we will manage it with the resource below - # This is only intended to be used in scenarios where the configmap does not exist - ignore_changes = [data, metadata[0].labels, metadata[0].annotations] - } -} - -resource "kubernetes_config_map_v1_data" "aws_auth" { - count = var.create && var.manage_aws_auth_configmap ? 1 : 0 - - force = true - - metadata { - name = "aws-auth" - namespace = "kube-system" - } - - data = local.aws_auth_configmap_data - - depends_on = [ - # Required for instances where the configmap does not exist yet to avoid race condition - kubernetes_config_map.aws_auth, - ] -} diff --git a/modules/aws-auth/outputs.tf b/modules/aws-auth/outputs.tf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/modules/aws-auth/variables.tf b/modules/aws-auth/variables.tf deleted file mode 100644 index 3aaeb023e3..0000000000 --- a/modules/aws-auth/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -variable "create" { - description = "Controls if resources should be created (affects all resources)" - type = bool - default = true -} - -################################################################################ -# aws-auth ConfigMap -################################################################################ - -variable "create_aws_auth_configmap" { - description = "Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap`" - type = bool - default = false -} - -variable "manage_aws_auth_configmap" { - description = "Determines whether to manage the aws-auth configmap" - type = bool - default = true -} - -variable "aws_auth_roles" { - description = "List of role maps to add to the aws-auth configmap" - type = list(any) - default = [] -} - -variable "aws_auth_users" { - description = "List of user maps to add to the aws-auth configmap" - type = list(any) - default = [] -} - -variable "aws_auth_accounts" { - description = "List of account maps to add to the aws-auth configmap" - type = list(any) - default = [] -} diff --git a/modules/aws-auth/versions.tf b/modules/aws-auth/versions.tf deleted file mode 100644 index f330045476..0000000000 --- a/modules/aws-auth/versions.tf +++ /dev/null @@ -1,10 +0,0 @@ -terraform { - required_version = ">= 1.3.2" - - required_providers { - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.20" - } - } -} diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 2eeb0c61c8..05eacc433c 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -63,14 +63,14 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws](#provider\_aws) | >= 6.0 | ## Modules @@ -82,7 +82,6 @@ module "eks_managed_node_group" { | Name | Type | |------|------| -| [aws_autoscaling_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_schedule) | resource | | [aws_eks_node_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | @@ -90,71 +89,72 @@ module "eks_managed_node_group" { | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | | [aws_placement_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/placement_group) | resource | +| [aws_security_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_vpc_security_group_egress_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_egress_rule) | resource | +| [aws_vpc_security_group_ingress_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | -| [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | +| [aws_eks_cluster_versions.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_versions) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_ssm_parameter.ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | -| [aws_subnets.placement_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | +| [aws_subnet.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [account\_id](#input\_account\_id) | The AWS account ID - pass through value to reduce number of GET requests from data sources | `string` | `""` | no | | [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance. If not supplied, EKS will use its own default image | `string` | `""` | no | | [ami\_release\_version](#input\_ami\_release\_version) | The AMI version. Defaults to latest AMI release version for the given Kubernetes version and AMI type | `string` | `null` | no | -| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `null` | no | -| [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | -| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | -| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `"AL2023_x86_64_STANDARD"` | no | +| [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI |
map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
}))
| `null` | no | +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `null` | no | +| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations |
object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
})
| `null` | no | | [capacity\_type](#input\_capacity\_type) | Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT` | `string` | `"ON_DEMAND"` | no | -| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | -| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | -| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | -| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `null` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `null` | no | +| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `null` | no | +| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `null` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | -| [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | +| [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `""` | no | | [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | -| [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | -| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | [Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes version. Defaults to EKS Cluster Kubernetes version | `string` | `null` | no | -| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | +| [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `null` | no | +| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance |
object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
})
| `null` | no | | [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_iam\_role\_policy](#input\_create\_iam\_role\_policy) | Determines whether an IAM role policy is created or not | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | | [create\_placement\_group](#input\_create\_placement\_group) | Determines whether a placement group is created & used by the node group | `bool` | `false` | no | -| [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | -| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | +| [create\_security\_group](#input\_create\_security\_group) | Determines if a security group is created | `bool` | `true` | no | +| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance |
object({
cpu_credits = optional(string)
})
| `null` | no | | [desired\_size](#input\_desired\_size) | Desired number of instances/nodes | `number` | `1` | no | | [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | | [disk\_size](#input\_disk\_size) | Disk size in GiB for nodes. Defaults to `20`. Only valid when `use_custom_launch_template` = `false` | `number` | `null` | no | | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance(s) will be EBS-optimized | `bool` | `null` | no | | [efa\_indices](#input\_efa\_indices) | The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true` | `list(number)` |
[
0
]
| no | -| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | -| [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | | [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template. Only valid when using a custom AMI via `ami_id` | `bool` | `false` | no | -| [enable\_efa\_only](#input\_enable\_efa\_only) | Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later | `bool` | `false` | no | +| [enable\_efa\_only](#input\_enable\_efa\_only) | Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later | `bool` | `true` | no | | [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | -| [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | -| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | +| [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `false` | no | +| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances |
object({
enabled = optional(bool)
})
| `null` | no | | [force\_update\_version](#input\_force\_update\_version) | Force version update if existing pods are unable to be drained due to a pod disruption budget issue | `bool` | `null` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the node group. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | -| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `"EKS managed node group IAM role"` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | -| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed |
list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
}))
| `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | -| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `{}` | no | +| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance |
object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
})
| `null` | no | | [instance\_types](#input\_instance\_types) | Set of instance types associated with the EKS Node Group. Defaults to `["t3.medium"]` | `list(string)` | `null` | no | | [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | | [key\_name](#input\_key\_name) | The key name that should be used for the instance(s) | `string` | `null` | no | +| [kubernetes\_version](#input\_kubernetes\_version) | Kubernetes version. Defaults to EKS Cluster Kubernetes version | `string` | `null` | no | | [labels](#input\_labels) | Key-value map of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed | `map(string)` | `null` | no | | [launch\_template\_default\_version](#input\_launch\_template\_default\_version) | Default version of the launch template | `string` | `null` | no | | [launch\_template\_description](#input\_launch\_template\_description) | Description of the launch template | `string` | `null` | no | @@ -163,42 +163,45 @@ module "eks_managed_node_group" { | [launch\_template\_tags](#input\_launch\_template\_tags) | A map of additional tags to add to the tag\_specifications of launch template created | `map(string)` | `{}` | no | | [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | | [launch\_template\_version](#input\_launch\_template\_version) | Launch template version number. The default is `$Default` | `string` | `null` | no | -| [license\_specifications](#input\_license\_specifications) | A map of license specifications to associate with | `any` | `{}` | no | -| [maintenance\_options](#input\_maintenance\_options) | The maintenance options for the instance | `any` | `{}` | no | +| [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with |
list(object({
license_configuration_arn = string
}))
| `null` | no | +| [maintenance\_options](#input\_maintenance\_options) | The maintenance options for the instance |
object({
auto_recovery = optional(string)
})
| `null` | no | | [max\_size](#input\_max\_size) | Maximum number of instances/nodes | `number` | `3` | no | -| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | -| [min\_size](#input\_min\_size) | Minimum number of instances/nodes | `number` | `0` | no | +| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance |
object({
http_endpoint = optional(string, "enabled")
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number, 1)
http_tokens = optional(string, "required")
instance_metadata_tags = optional(string)
})
|
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 1,
"http_tokens": "required"
}
| no | +| [min\_size](#input\_min\_size) | Minimum number of instances/nodes | `number` | `1` | no | | [name](#input\_name) | Name of the EKS managed node group | `string` | `""` | no | -| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | +| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time |
list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
subnet_id = optional(string)
}))
| `[]` | no | | [node\_repair\_config](#input\_node\_repair\_config) | The node auto repair configuration for the node group |
object({
enabled = optional(bool, true)
})
| `null` | no | -| [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | -| [placement\_group\_az](#input\_placement\_group\_az) | Availability zone where placement group is created (ex. `eu-west-1c`) | `string` | `null` | no | -| [placement\_group\_strategy](#input\_placement\_group\_strategy) | The placement group strategy | `string` | `"cluster"` | no | -| [platform](#input\_platform) | [DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | `string` | `"linux"` | no | -| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | -| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | -| [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet | `map(string)` | `{}` | no | +| [partition](#input\_partition) | The AWS partition - pass through value to reduce number of GET requests from data sources | `string` | `""` | no | +| [placement](#input\_placement) | The placement of the instance |
object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
})
| `null` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `null` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `null` | no | +| [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet |
object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
})
| `null` | no | | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | -| [remote\_access](#input\_remote\_access) | Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false` | `any` | `{}` | no | -| [schedules](#input\_schedules) | Map of autoscaling group schedule to create | `map(any)` | `{}` | no | +| [region](#input\_region) | Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration | `string` | `null` | no | +| [remote\_access](#input\_remote\_access) | Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false` |
object({
ec2_ssh_key = optional(string)
source_security_group_ids = optional(list(string))
})
| `null` | no | +| [security\_group\_description](#input\_security\_group\_description) | Description of the security group created | `string` | `null` | no | +| [security\_group\_egress\_rules](#input\_security\_group\_egress\_rules) | Security group egress rules to add to the security group created |
map(object({
name = optional(string)

cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string, "tcp")
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool, false)
tags = optional(map(string), {})
to_port = optional(string)
}))
| `{}` | no | +| [security\_group\_ingress\_rules](#input\_security\_group\_ingress\_rules) | Security group ingress rules to add to the security group created |
map(object({
name = optional(string)

cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string, "tcp")
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool, false)
tags = optional(map(string), {})
to_port = optional(string)
}))
| `{}` | no | +| [security\_group\_name](#input\_security\_group\_name) | Name to use on security group created | `string` | `null` | no | +| [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the security group created | `map(string)` | `{}` | no | +| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `bool` | `true` | no | | [subnet\_ids](#input\_subnet\_ids) | Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME` | `list(string)` | `null` | no | | [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | -| [taints](#input\_taints) | The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group | `any` | `{}` | no | -| [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the node group | `map(string)` | `{}` | no | -| [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates | `map(string)` |
{
"max_unavailable_percentage": 33
}
| no | +| [taints](#input\_taints) | The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group |
map(object({
key = string
value = optional(string)
effect = string
}))
| `null` | no | +| [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the node group |
object({
create = optional(string)
update = optional(string)
delete = optional(string)
})
| `null` | no | +| [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates |
object({
max_unavailable = optional(number)
max_unavailable_percentage = optional(number)
})
|
{
"max_unavailable_percentage": 33
}
| no | | [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update the launch templates default version on each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | | [use\_custom\_launch\_template](#input\_use\_custom\_launch\_template) | Determines whether to use a custom launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | -| [use\_latest\_ami\_release\_version](#input\_use\_latest\_ami\_release\_version) | Determines whether to use the latest AMI release version for the given `ami_type` (except for `CUSTOM`). Note: `ami_type` and `cluster_version` must be supplied in order to enable this feature | `bool` | `false` | no | +| [use\_latest\_ami\_release\_version](#input\_use\_latest\_ami\_release\_version) | Determines whether to use the latest AMI release version for the given `ami_type` (except for `CUSTOM`). Note: `ami_type` and `kubernetes_version` must be supplied in order to enable this feature | `bool` | `true` | no | | [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix | `bool` | `true` | no | -| [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | +| [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `null` | no | | [vpc\_security\_group\_ids](#input\_vpc\_security\_group\_ids) | A list of security group IDs to associate | `list(string)` | `[]` | no | ## Outputs | Name | Description | |------|-------------| -| [autoscaling\_group\_schedule\_arns](#output\_autoscaling\_group\_schedule\_arns) | ARNs of autoscaling group schedules | | [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | | [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | | [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | @@ -213,5 +216,6 @@ module "eks_managed_node_group" { | [node\_group\_resources](#output\_node\_group\_resources) | List of objects containing information about underlying resources | | [node\_group\_status](#output\_node\_group\_status) | Status of the EKS Node Group | | [node\_group\_taints](#output\_node\_group\_taints) | List of objects containing information about taints applied to the node group | -| [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | +| [security\_group\_arn](#output\_security\_group\_arn) | Amazon Resource Name (ARN) of the security group | +| [security\_group\_id](#output\_security\_group\_id) | ID of the security group | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index b9c9ba0e87..d07c38633d 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -1,5 +1,14 @@ -data "aws_partition" "current" {} -data "aws_caller_identity" "current" {} +data "aws_partition" "current" { + count = var.create && var.partition == "" ? 1 : 0 +} +data "aws_caller_identity" "current" { + count = var.create && var.account_id == "" ? 1 : 0 +} + +locals { + partition = try(data.aws_partition.current[0].partition, var.partition) + account_id = try(data.aws_caller_identity.current[0].account_id, var.account_id) +} ################################################################################ # User Data @@ -9,14 +18,13 @@ module "user_data" { source = "../_user_data" create = var.create - platform = var.platform ami_type = var.ami_type cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 cluster_ip_family = var.cluster_ip_family - cluster_service_cidr = try(coalesce(var.cluster_service_cidr, var.cluster_service_ipv4_cidr), "") + cluster_service_cidr = var.cluster_service_cidr enable_bootstrap_user_data = var.enable_bootstrap_user_data pre_bootstrap_user_data = var.pre_bootstrap_user_data @@ -35,6 +43,8 @@ module "user_data" { data "aws_ec2_instance_type" "this" { count = var.create && var.enable_efa_support ? 1 : 0 + region = var.region + instance_type = local.efa_instance_type } @@ -64,71 +74,72 @@ locals { locals { launch_template_name = coalesce(var.launch_template_name, "${var.name}-eks-node-group") - security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) - - placement = local.create_placement_group ? { group_name = aws_placement_group.this[0].name } : var.placement + security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids, aws_security_group.this[*].id)) } resource "aws_launch_template" "this" { count = var.create && var.create_launch_template && var.use_custom_launch_template ? 1 : 0 + region = var.region + dynamic "block_device_mappings" { - for_each = var.block_device_mappings + for_each = var.block_device_mappings != null ? var.block_device_mappings : {} content { - device_name = try(block_device_mappings.value.device_name, null) + device_name = block_device_mappings.value.device_name dynamic "ebs" { - for_each = try([block_device_mappings.value.ebs], []) + for_each = block_device_mappings.value.ebs != null ? [block_device_mappings.value.ebs] : [] content { - delete_on_termination = try(ebs.value.delete_on_termination, null) - encrypted = try(ebs.value.encrypted, null) - iops = try(ebs.value.iops, null) - kms_key_id = try(ebs.value.kms_key_id, null) - snapshot_id = try(ebs.value.snapshot_id, null) - throughput = try(ebs.value.throughput, null) - volume_size = try(ebs.value.volume_size, null) - volume_type = try(ebs.value.volume_type, null) + delete_on_termination = ebs.value.delete_on_termination + encrypted = ebs.value.encrypted + iops = ebs.value.iops + kms_key_id = ebs.value.kms_key_id + snapshot_id = ebs.value.snapshot_id + throughput = ebs.value.throughput + volume_initialization_rate = ebs.value.volume_initialization_rate + volume_size = ebs.value.volume_size + volume_type = ebs.value.volume_type } } - no_device = try(block_device_mappings.value.no_device, null) - virtual_name = try(block_device_mappings.value.virtual_name, null) + no_device = block_device_mappings.value.no_device + virtual_name = block_device_mappings.value.virtual_name } } dynamic "capacity_reservation_specification" { - for_each = length(var.capacity_reservation_specification) > 0 ? [var.capacity_reservation_specification] : [] + for_each = var.capacity_reservation_specification != null ? [var.capacity_reservation_specification] : [] content { - capacity_reservation_preference = try(capacity_reservation_specification.value.capacity_reservation_preference, null) + capacity_reservation_preference = capacity_reservation_specification.value.capacity_reservation_preference dynamic "capacity_reservation_target" { - for_each = try([capacity_reservation_specification.value.capacity_reservation_target], []) - + for_each = capacity_reservation_specification.value.capacity_reservation_target != null ? [capacity_reservation_specification.value.capacity_reservation_target] : [] content { - capacity_reservation_id = try(capacity_reservation_target.value.capacity_reservation_id, null) - capacity_reservation_resource_group_arn = try(capacity_reservation_target.value.capacity_reservation_resource_group_arn, null) + capacity_reservation_id = capacity_reservation_target.value.capacity_reservation_id + capacity_reservation_resource_group_arn = capacity_reservation_target.value.capacity_reservation_resource_group_arn } } } } dynamic "cpu_options" { - for_each = length(var.cpu_options) > 0 ? [var.cpu_options] : [] + for_each = var.cpu_options != null ? [var.cpu_options] : [] content { - core_count = try(cpu_options.value.core_count, null) - threads_per_core = try(cpu_options.value.threads_per_core, null) + amd_sev_snp = cpu_options.value.amd_sev_snp + core_count = cpu_options.value.core_count + threads_per_core = cpu_options.value.threads_per_core } } dynamic "credit_specification" { - for_each = length(var.credit_specification) > 0 ? [var.credit_specification] : [] + for_each = var.credit_specification != null ? [var.credit_specification] : [] content { - cpu_credits = try(credit_specification.value.cpu_credits, null) + cpu_credits = credit_specification.value.cpu_credits } } @@ -137,24 +148,8 @@ resource "aws_launch_template" "this" { disable_api_termination = var.disable_api_termination ebs_optimized = var.ebs_optimized - dynamic "elastic_gpu_specifications" { - for_each = var.elastic_gpu_specifications - - content { - type = elastic_gpu_specifications.value.type - } - } - - dynamic "elastic_inference_accelerator" { - for_each = length(var.elastic_inference_accelerator) > 0 ? [var.elastic_inference_accelerator] : [] - - content { - type = elastic_inference_accelerator.value.type - } - } - dynamic "enclave_options" { - for_each = length(var.enclave_options) > 0 ? [var.enclave_options] : [] + for_each = var.enclave_options != null ? [var.enclave_options] : [] content { enabled = enclave_options.value.enabled @@ -187,20 +182,20 @@ resource "aws_launch_template" "this" { # instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior dynamic "instance_market_options" { - for_each = length(var.instance_market_options) > 0 ? [var.instance_market_options] : [] + for_each = var.instance_market_options != null ? [var.instance_market_options] : [] content { - market_type = try(instance_market_options.value.market_type, null) + market_type = instance_market_options.value.market_type dynamic "spot_options" { - for_each = try([instance_market_options.value.spot_options], []) + for_each = instance_market_options.value.spot_options != null ? [instance_market_options.value.spot_options] : [] content { - block_duration_minutes = try(spot_options.value.block_duration_minutes, null) - instance_interruption_behavior = try(spot_options.value.instance_interruption_behavior, null) - max_price = try(spot_options.value.max_price, null) - spot_instance_type = try(spot_options.value.spot_instance_type, null) - valid_until = try(spot_options.value.valid_until, null) + block_duration_minutes = spot_options.value.block_duration_minutes + instance_interruption_behavior = spot_options.value.instance_interruption_behavior + max_price = spot_options.value.max_price + spot_instance_type = spot_options.value.spot_instance_type + valid_until = spot_options.value.valid_until } } } @@ -213,7 +208,7 @@ resource "aws_launch_template" "this" { key_name = var.key_name dynamic "license_specification" { - for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} + for_each = var.license_specifications != null ? var.license_specifications : [] content { license_configuration_arn = license_specification.value.license_configuration_arn @@ -221,22 +216,22 @@ resource "aws_launch_template" "this" { } dynamic "maintenance_options" { - for_each = length(var.maintenance_options) > 0 ? [var.maintenance_options] : [] + for_each = var.maintenance_options != null ? [var.maintenance_options] : [] content { - auto_recovery = try(maintenance_options.value.auto_recovery, null) + auto_recovery = maintenance_options.value.auto_recovery } } dynamic "metadata_options" { - for_each = length(var.metadata_options) > 0 ? [var.metadata_options] : [] + for_each = var.metadata_options != null ? [var.metadata_options] : [] content { - http_endpoint = try(metadata_options.value.http_endpoint, null) - http_protocol_ipv6 = try(metadata_options.value.http_protocol_ipv6, null) - http_put_response_hop_limit = try(metadata_options.value.http_put_response_hop_limit, null) - http_tokens = try(metadata_options.value.http_tokens, null) - instance_metadata_tags = try(metadata_options.value.instance_metadata_tags, null) + http_endpoint = metadata_options.value.http_endpoint + http_protocol_ipv6 = metadata_options.value.http_protocol_ipv6 + http_put_response_hop_limit = metadata_options.value.http_put_response_hop_limit + http_tokens = metadata_options.value.http_tokens + instance_metadata_tags = metadata_options.value.instance_metadata_tags } } @@ -252,29 +247,57 @@ resource "aws_launch_template" "this" { name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null dynamic "network_interfaces" { - for_each = local.network_interfaces + for_each = length(var.network_interfaces) > 0 ? var.network_interfaces : [] content { - associate_carrier_ip_address = try(network_interfaces.value.associate_carrier_ip_address, null) - associate_public_ip_address = try(network_interfaces.value.associate_public_ip_address, null) - delete_on_termination = try(network_interfaces.value.delete_on_termination, null) - description = try(network_interfaces.value.description, null) - device_index = try(network_interfaces.value.device_index, null) - interface_type = try(network_interfaces.value.interface_type, null) - ipv4_address_count = try(network_interfaces.value.ipv4_address_count, null) - ipv4_addresses = try(network_interfaces.value.ipv4_addresses, []) - ipv4_prefix_count = try(network_interfaces.value.ipv4_prefix_count, null) - ipv4_prefixes = try(network_interfaces.value.ipv4_prefixes, null) - ipv6_address_count = try(network_interfaces.value.ipv6_address_count, null) - ipv6_addresses = try(network_interfaces.value.ipv6_addresses, []) - ipv6_prefix_count = try(network_interfaces.value.ipv6_prefix_count, null) - ipv6_prefixes = try(network_interfaces.value.ipv6_prefixes, []) - network_card_index = try(network_interfaces.value.network_card_index, null) - network_interface_id = try(network_interfaces.value.network_interface_id, null) - primary_ipv6 = try(network_interfaces.value.primary_ipv6, null) - private_ip_address = try(network_interfaces.value.private_ip_address, null) + associate_carrier_ip_address = network_interfaces.value.associate_carrier_ip_address + associate_public_ip_address = network_interfaces.value.associate_public_ip_address + + dynamic "connection_tracking_specification" { + for_each = network_interfaces.value.connection_tracking_specification != null ? [network_interfaces.value.connection_tracking_specification] : [] + + content { + tcp_established_timeout = connection_tracking_specification.value.tcp_established_timeout + udp_stream_timeout = connection_tracking_specification.value.udp_stream_timeout + udp_timeout = connection_tracking_specification.value.udp_timeout + } + } + + delete_on_termination = network_interfaces.value.delete_on_termination + description = network_interfaces.value.description + device_index = network_interfaces.value.device_index + + dynamic "ena_srd_specification" { + for_each = network_interfaces.value.ena_srd_specification != null ? [network_interfaces.value.ena_srd_specification] : [] + + content { + ena_srd_enabled = ena_srd_specification.value.ena_srd_enabled + + dynamic "ena_srd_udp_specification" { + for_each = ena_srd_specification.value.ena_srd_udp_specification != null ? [ena_srd_specification.value.ena_srd_udp_specification] : [] + + content { + ena_srd_udp_enabled = ena_srd_udp_specification.value.ena_srd_udp_enabled + } + } + } + } + + interface_type = network_interfaces.value.interface_type + ipv4_address_count = network_interfaces.value.ipv4_address_count + ipv4_addresses = network_interfaces.value.ipv4_addresses + ipv4_prefix_count = network_interfaces.value.ipv4_prefix_count + ipv4_prefixes = network_interfaces.value.ipv4_prefixes + ipv6_address_count = network_interfaces.value.ipv6_address_count + ipv6_addresses = network_interfaces.value.ipv6_addresses + ipv6_prefix_count = network_interfaces.value.ipv6_prefix_count + ipv6_prefixes = network_interfaces.value.ipv6_prefixes + network_card_index = network_interfaces.value.network_card_index + network_interface_id = network_interfaces.value.network_interface_id + primary_ipv6 = network_interfaces.value.primary_ipv6 + private_ip_address = network_interfaces.value.private_ip_address # Ref: https://github.com/hashicorp/terraform-provider-aws/issues/4570 - security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) + security_groups = compact(concat(network_interfaces.value.security_groups, var.vpc_security_group_ids)) # Set on EKS managed node group, will fail if set here # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics # subnet_id = try(network_interfaces.value.subnet_id, null) @@ -282,14 +305,14 @@ resource "aws_launch_template" "this" { } dynamic "placement" { - for_each = length(local.placement) > 0 ? [local.placement] : [] + for_each = var.placement != null || local.create_placement_group ? [var.placement] : [] content { affinity = try(placement.value.affinity, null) - availability_zone = lookup(placement.value, "availability_zone", null) - group_name = lookup(placement.value, "group_name", null) - host_id = lookup(placement.value, "host_id", null) - host_resource_group_arn = lookup(placement.value, "host_resource_group_arn", null) + availability_zone = try(placement.value.availability_zone, null) + group_name = try(aws_placement_group.this[0].name, placement.value.group_name) + host_id = try(placement.value.host_id, null) + host_resource_group_arn = try(placement.value.host_resource_group_arn, null) partition_number = try(placement.value.partition_number, null) spread_domain = try(placement.value.spread_domain, null) tenancy = try(placement.value.tenancy, null) @@ -297,12 +320,12 @@ resource "aws_launch_template" "this" { } dynamic "private_dns_name_options" { - for_each = length(var.private_dns_name_options) > 0 ? [var.private_dns_name_options] : [] + for_each = var.private_dns_name_options != null ? [var.private_dns_name_options] : [] content { - enable_resource_name_dns_aaaa_record = try(private_dns_name_options.value.enable_resource_name_dns_aaaa_record, null) - enable_resource_name_dns_a_record = try(private_dns_name_options.value.enable_resource_name_dns_a_record, null) - hostname_type = try(private_dns_name_options.value.hostname_type, null) + enable_resource_name_dns_aaaa_record = private_dns_name_options.value.enable_resource_name_dns_aaaa_record + enable_resource_name_dns_a_record = private_dns_name_options.value.enable_resource_name_dns_a_record + hostname_type = private_dns_name_options.value.hostname_type } } @@ -342,32 +365,41 @@ resource "aws_launch_template" "this" { # AMI SSM Parameter ################################################################################ +data "aws_eks_cluster_versions" "this" { + count = var.create && var.kubernetes_version == null ? 1 : 0 + + region = var.region + + cluster_type = "eks" + version_status = "STANDARD_SUPPORT" +} + locals { # Just to ensure templating doesn't fail when values are not provided - ssm_cluster_version = var.cluster_version != null ? var.cluster_version : "" - ssm_ami_type = var.ami_type != null ? var.ami_type : "" + ssm_kubernetes_version = var.kubernetes_version != null ? var.kubernetes_version : try(data.aws_eks_cluster_versions.this[0].cluster_versions[0].cluster_version, "UNSPECIFIED") + ssm_ami_type = var.ami_type != null ? var.ami_type : "" # Map the AMI type to the respective SSM param path ssm_ami_type_to_ssm_param = { - AL2_x86_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2/recommended/release_version" - AL2_x86_64_GPU = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-gpu/recommended/release_version" - AL2_ARM_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-arm64/recommended/release_version" + AL2_x86_64 = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2/recommended/release_version" + AL2_x86_64_GPU = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2-gpu/recommended/release_version" + AL2_ARM_64 = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2-arm64/recommended/release_version" CUSTOM = "NONE" - BOTTLEROCKET_ARM_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/arm64/latest/image_version" - BOTTLEROCKET_x86_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/x86_64/latest/image_version" - BOTTLEROCKET_ARM_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/arm64/latest/image_version" - BOTTLEROCKET_x86_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/x86_64/latest/image_version" - BOTTLEROCKET_ARM_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/arm64/latest/image_version" - BOTTLEROCKET_x86_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/x86_64/latest/image_version" - WINDOWS_CORE_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-${local.ssm_cluster_version}" - WINDOWS_FULL_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-${local.ssm_cluster_version}" - WINDOWS_CORE_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-EKS_Optimized-${local.ssm_cluster_version}" - WINDOWS_FULL_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-${local.ssm_cluster_version}" - AL2023_x86_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/standard/recommended/release_version" - AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/standard/recommended/release_version" - AL2023_x86_64_NEURON = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/neuron/recommended/release_version" - AL2023_x86_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/nvidia/recommended/release_version" - AL2023_ARM_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/nvidia/recommended/release_version" + BOTTLEROCKET_ARM_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}/arm64/latest/image_version" + BOTTLEROCKET_x86_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}/x86_64/latest/image_version" + BOTTLEROCKET_ARM_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}-fips/arm64/latest/image_version" + BOTTLEROCKET_x86_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}-fips/x86_64/latest/image_version" + BOTTLEROCKET_ARM_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}-nvidia/arm64/latest/image_version" + BOTTLEROCKET_x86_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}-nvidia/x86_64/latest/image_version" + WINDOWS_CORE_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-${local.ssm_kubernetes_version}" + WINDOWS_FULL_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-${local.ssm_kubernetes_version}" + WINDOWS_CORE_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-EKS_Optimized-${local.ssm_kubernetes_version}" + WINDOWS_FULL_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-${local.ssm_kubernetes_version}" + AL2023_x86_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2023/x86_64/standard/recommended/release_version" + AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2023/arm64/standard/recommended/release_version" + AL2023_x86_64_NEURON = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2023/x86_64/neuron/recommended/release_version" + AL2023_x86_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2023/x86_64/nvidia/recommended/release_version" + AL2023_ARM_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2023/arm64/nvidia/recommended/release_version" } # The Windows SSM params currently do not have a release version, so we have to get the full output JSON blob and parse out the release version @@ -379,6 +411,8 @@ locals { data "aws_ssm_parameter" "ami" { count = var.create && var.use_latest_ami_release_version ? 1 : 0 + region = var.region + name = local.ssm_ami_type_to_ssm_param[var.ami_type] } @@ -395,10 +429,12 @@ locals { resource "aws_eks_node_group" "this" { count = var.create ? 1 : 0 + region = var.region + # Required cluster_name = var.cluster_name node_role_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn - subnet_ids = local.create_placement_group ? data.aws_subnets.placement_group[0].ids : var.subnet_ids + subnet_ids = var.subnet_ids scaling_config { min_size = var.min_size @@ -413,7 +449,7 @@ resource "aws_eks_node_group" "this" { # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami ami_type = var.ami_id != "" ? null : var.ami_type release_version = var.ami_id != "" ? null : var.use_latest_ami_release_version ? local.latest_ami_release_version : var.ami_release_version - version = var.ami_id != "" ? null : var.cluster_version + version = var.ami_id != "" ? null : var.kubernetes_version capacity_type = var.capacity_type disk_size = var.use_custom_launch_template ? null : var.disk_size # if using a custom LT, set disk size on custom LT or else it will error here @@ -432,30 +468,30 @@ resource "aws_eks_node_group" "this" { } dynamic "remote_access" { - for_each = length(var.remote_access) > 0 ? [var.remote_access] : [] + for_each = var.remote_access != null ? [var.remote_access] : [] content { - ec2_ssh_key = try(remote_access.value.ec2_ssh_key, null) - source_security_group_ids = try(remote_access.value.source_security_group_ids, []) + ec2_ssh_key = remote_access.value.ec2_ssh_key + source_security_group_ids = remote_access.value.source_security_group_ids } } dynamic "taint" { - for_each = var.taints + for_each = var.taints != null ? var.taints : {} content { key = taint.value.key - value = try(taint.value.value, null) + value = taint.value.value effect = taint.value.effect } } dynamic "update_config" { - for_each = length(var.update_config) > 0 ? [var.update_config] : [] + for_each = var.update_config != null ? [var.update_config] : [] content { - max_unavailable_percentage = try(update_config.value.max_unavailable_percentage, null) - max_unavailable = try(update_config.value.max_unavailable, null) + max_unavailable_percentage = update_config.value.max_unavailable_percentage + max_unavailable = update_config.value.max_unavailable } } @@ -467,10 +503,14 @@ resource "aws_eks_node_group" "this" { } } - timeouts { - create = lookup(var.timeouts, "create", null) - update = lookup(var.timeouts, "update", null) - delete = lookup(var.timeouts, "delete", null) + dynamic "timeouts" { + for_each = var.timeouts != null ? [var.timeouts] : [] + + content { + create = var.timeouts.create + update = var.timeouts.update + delete = var.timeouts.delete + } } lifecycle { @@ -494,13 +534,13 @@ locals { create_iam_role = var.create && var.create_iam_role iam_role_name = coalesce(var.iam_role_name, "${var.name}-eks-node-group") - iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" ipv4_cni_policy = { for k, v in { AmazonEKS_CNI_Policy = "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } ipv6_cni_policy = { for k, v in { - AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + AmazonEKS_CNI_IPv6_Policy = "arn:${local.partition}:iam::${local.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } } @@ -560,25 +600,25 @@ resource "aws_iam_role_policy_attachment" "additional" { ################################################################################ locals { - create_iam_role_policy = local.create_iam_role && var.create_iam_role_policy && length(var.iam_role_policy_statements) > 0 + create_iam_role_policy = local.create_iam_role && var.create_iam_role_policy && var.iam_role_policy_statements != null } data "aws_iam_policy_document" "role" { count = local.create_iam_role_policy ? 1 : 0 dynamic "statement" { - for_each = var.iam_role_policy_statements + for_each = var.iam_role_policy_statements != null ? var.iam_role_policy_statements : [] content { - sid = try(statement.value.sid, null) - actions = try(statement.value.actions, null) - not_actions = try(statement.value.not_actions, null) - effect = try(statement.value.effect, null) - resources = try(statement.value.resources, null) - not_resources = try(statement.value.not_resources, null) + sid = statement.value.sid + actions = statement.value.actions + not_actions = statement.value.not_actions + effect = statement.value.effect + resources = statement.value.resources + not_resources = statement.value.not_resources dynamic "principals" { - for_each = try(statement.value.principals, []) + for_each = statement.value.principals != null ? statement.value.principals : [] content { type = principals.value.type @@ -587,7 +627,7 @@ data "aws_iam_policy_document" "role" { } dynamic "not_principals" { - for_each = try(statement.value.not_principals, []) + for_each = statement.value.not_principals != null ? statement.value.not_principals : [] content { type = not_principals.value.type @@ -596,7 +636,7 @@ data "aws_iam_policy_document" "role" { } dynamic "condition" { - for_each = try(statement.value.conditions, []) + for_each = statement.value.condition != null ? statement.value.condition : [] content { test = condition.value.test @@ -628,84 +668,117 @@ locals { resource "aws_placement_group" "this" { count = local.create_placement_group ? 1 : 0 + region = var.region + name = "${var.cluster_name}-${var.name}" - strategy = var.placement_group_strategy + strategy = "cluster" tags = var.tags } ################################################################################ -# Instance AZ Lookup - -# Instances usually used in placement groups w/ EFA are only available in -# select availability zones. These data sources will cross reference the availability -# zones supported by the instance type with the subnets provided to ensure only -# AZs/subnets that are supported are used. +# Security Group ################################################################################ -# Find the availability zones supported by the instance type -# TODO - remove at next breaking change -# Force users to be explicit about which AZ to use when using placement groups, -# with or without EFA support -data "aws_ec2_instance_type_offerings" "this" { - count = local.enable_efa_support ? 1 : 0 - - filter { - name = "instance-type" - values = [local.efa_instance_type] - } +locals { + create_security_group = var.create && var.create_security_group && length(merge(local.security_group_ingress_rules, local.security_group_egress_rules)) > 0 + security_group_name = coalesce(var.security_group_name, "${var.cluster_name}-${var.name}") - location_type = "availability-zone-id" + security_group_ingress_rules = merge({ for k, v in + { + all_self_efa = { + description = "Node to node EFA" + protocol = "-1" + from_port = 0 + self = true + } + } : k => v if var.enable_efa_support + }, + var.security_group_ingress_rules + ) + security_group_egress_rules = merge({ for k, v in + { + all_self_efa = { + description = "Node to node EFA" + protocol = "-1" + to_port = 0 + self = true + } + } : k => v if var.enable_efa_support + }, + var.security_group_egress_rules + ) } -# Reverse the lookup to find one of the subnets provided based on the availability -# availability zone ID of the queried instance type (supported) -data "aws_subnets" "placement_group" { - count = local.create_placement_group ? 1 : 0 +data "aws_subnet" "this" { + count = local.create_security_group ? 1 : 0 - filter { - name = "subnet-id" - values = var.subnet_ids - } + region = var.region - # The data source can lookup the first available AZ or you can specify an AZ (next filter) - dynamic "filter" { - for_each = var.enable_efa_support && var.placement_group_az == null ? [1] : [] + id = element(var.subnet_ids, 0) +} - content { - name = "availability-zone-id" - values = data.aws_ec2_instance_type_offerings.this[0].locations - } - } +resource "aws_security_group" "this" { + count = local.create_security_group ? 1 : 0 - dynamic "filter" { - for_each = var.placement_group_az != null ? [var.placement_group_az] : [] + region = var.region - content { - name = "availability-zone" - values = [filter.value] - } + name = var.security_group_use_name_prefix ? null : local.security_group_name + name_prefix = var.security_group_use_name_prefix ? "${local.security_group_name}-" : null + description = var.security_group_description + vpc_id = data.aws_subnet.this[0].vpc_id + + tags = merge( + var.tags, + { "Name" = local.security_group_name }, + var.security_group_tags + ) + + lifecycle { + create_before_destroy = true } } -################################################################################ -# Autoscaling Group Schedule -################################################################################ +resource "aws_vpc_security_group_ingress_rule" "this" { + for_each = { for k, v in local.security_group_ingress_rules : k => v if length(local.security_group_ingress_rules) > 0 && local.create_security_group } + + region = var.region -resource "aws_autoscaling_schedule" "this" { - for_each = { for k, v in var.schedules : k => v if var.create && var.create_schedule } + cidr_ipv4 = each.value.cidr_ipv4 + cidr_ipv6 = each.value.cidr_ipv6 + description = each.value.description + from_port = each.value.from_port + ip_protocol = each.value.ip_protocol + prefix_list_id = each.value.prefix_list_id + referenced_security_group_id = each.value.self ? aws_security_group.this[0].id : each.value.referenced_security_group_id + security_group_id = aws_security_group.this[0].id + tags = merge( + var.tags, + var.security_group_tags, + { "Name" = coalesce(each.value.name, "${local.security_group_name}-${each.key}") }, + each.value.tags + ) + to_port = try(coalesce(each.value.to_port, each.value.from_port), null) +} - scheduled_action_name = each.key - autoscaling_group_name = aws_eks_node_group.this[0].resources[0].autoscaling_groups[0].name +resource "aws_vpc_security_group_egress_rule" "this" { + for_each = { for k, v in local.security_group_egress_rules : k => v if length(local.security_group_egress_rules) > 0 && local.create_security_group } - min_size = try(each.value.min_size, -1) - max_size = try(each.value.max_size, -1) - desired_capacity = try(each.value.desired_size, -1) - start_time = try(each.value.start_time, null) - end_time = try(each.value.end_time, null) - time_zone = try(each.value.time_zone, null) + region = var.region - # [Minute] [Hour] [Day_of_Month] [Month_of_Year] [Day_of_Week] - # Cron examples: https://crontab.guru/examples.html - recurrence = try(each.value.recurrence, null) + cidr_ipv4 = each.value.cidr_ipv4 + cidr_ipv6 = each.value.cidr_ipv6 + description = each.value.description + from_port = try(coalesce(each.value.from_port, each.value.to_port), null) + ip_protocol = each.value.ip_protocol + prefix_list_id = each.value.prefix_list_id + referenced_security_group_id = each.value.self ? aws_security_group.this[0].id : each.value.referenced_security_group_id + security_group_id = aws_security_group.this[0].id + tags = merge( + var.tags, + var.security_group_tags, + { "Name" = coalesce(each.value.name, "${local.security_group_name}-${each.key}") }, + each.value.tags + ) + to_port = each.value.to_port } diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index 8cab6e2c13..a7d6fcf62b 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -61,15 +61,6 @@ output "node_group_taints" { value = try(aws_eks_node_group.this[0].taint, []) } -################################################################################ -# Autoscaling Group Schedule -################################################################################ - -output "autoscaling_group_schedule_arns" { - description = "ARNs of autoscaling group schedules" - value = { for k, v in aws_autoscaling_schedule.this : k => v.arn } -} - ################################################################################ # IAM Role ################################################################################ @@ -90,10 +81,15 @@ output "iam_role_unique_id" { } ################################################################################ -# Additional +# Security Group ################################################################################ -output "platform" { - description = "[DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" - value = module.user_data.platform +output "security_group_arn" { + description = "Amazon Resource Name (ARN) of the security group" + value = try(aws_security_group.this[0].arn, null) +} + +output "security_group_id" { + description = "ID of the security group" + value = try(aws_security_group.this[0].id, null) } diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 54338542c9..281a32f63b 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -2,6 +2,7 @@ variable "create" { description = "Determines whether to create EKS managed node group or not" type = bool default = true + nullable = false } variable "tags" { @@ -10,10 +11,22 @@ variable "tags" { default = {} } -variable "platform" { - description = "[DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" +variable "region" { + description = "Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration" type = string - default = "linux" + default = null +} + +variable "partition" { + description = "The AWS partition - pass through value to reduce number of GET requests from data sources" + type = string + default = "" +} + +variable "account_id" { + description = "The AWS account ID - pass through value to reduce number of GET requests from data sources" + type = string + default = "" } ################################################################################ @@ -24,61 +37,55 @@ variable "enable_bootstrap_user_data" { description = "Determines whether the bootstrap configurations are populated within the user data template. Only valid when using a custom AMI via `ami_id`" type = bool default = false + nullable = false } variable "cluster_name" { description = "Name of associated EKS cluster" type = string - default = null + default = "" } variable "cluster_endpoint" { description = "Endpoint of associated EKS cluster" type = string - default = "" + default = null } variable "cluster_auth_base64" { description = "Base64 encoded CA of associated EKS cluster" type = string - default = "" + default = null } variable "cluster_service_cidr" { description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" type = string - default = "" -} - -# TODO - remove at next breaking change -variable "cluster_service_ipv4_cidr" { - description = "[Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" - type = string default = null } variable "pre_bootstrap_user_data" { description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string - default = "" + default = null } variable "post_bootstrap_user_data" { description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string - default = "" + default = null } variable "bootstrap_extra_args" { description = "Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" type = string - default = "" + default = null } variable "user_data_template_path" { description = "Path to a local, custom user data template file to use when rendering user data" type = string - default = "" + default = null } variable "cloudinit_pre_nodeadm" { @@ -89,7 +96,7 @@ variable "cloudinit_pre_nodeadm" { filename = optional(string) merge_type = optional(string) })) - default = [] + default = null } variable "cloudinit_post_nodeadm" { @@ -100,7 +107,7 @@ variable "cloudinit_post_nodeadm" { filename = optional(string) merge_type = optional(string) })) - default = [] + default = null } ################################################################################ @@ -111,12 +118,14 @@ variable "create_launch_template" { description = "Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template" type = bool default = true + nullable = false } variable "use_custom_launch_template" { description = "Determines whether to use a custom launch template or not. If set to `false`, EKS will use its own default launch template" type = bool default = true + nullable = false } variable "launch_template_id" { @@ -135,6 +144,7 @@ variable "launch_template_use_name_prefix" { description = "Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix" type = bool default = true + nullable = false } variable "launch_template_description" { @@ -165,6 +175,7 @@ variable "vpc_security_group_ids" { description = "A list of security group IDs to associate" type = list(string) default = [] + nullable = false } variable "cluster_primary_security_group_id" { @@ -183,6 +194,7 @@ variable "update_launch_template_default_version" { description = "Whether to update the launch templates default version on each update. Conflicts with `launch_template_default_version`" type = bool default = true + nullable = false } variable "disable_api_termination" { @@ -205,129 +217,208 @@ variable "ram_disk_id" { variable "block_device_mappings" { description = "Specify volumes to attach to the instance besides the volumes specified by the AMI" - type = any - default = {} + type = map(object({ + device_name = optional(string) + ebs = optional(object({ + delete_on_termination = optional(bool) + encrypted = optional(bool) + iops = optional(number) + kms_key_id = optional(string) + snapshot_id = optional(string) + throughput = optional(number) + volume_initialization_rate = optional(number) + volume_size = optional(number) + volume_type = optional(string) + })) + no_device = optional(string) + virtual_name = optional(string) + })) + default = null } variable "capacity_reservation_specification" { description = "Targeting for EC2 capacity reservations" - type = any - default = {} + type = object({ + capacity_reservation_preference = optional(string) + capacity_reservation_target = optional(object({ + capacity_reservation_id = optional(string) + capacity_reservation_resource_group_arn = optional(string) + })) + }) + default = null } variable "cpu_options" { description = "The CPU options for the instance" - type = map(string) - default = {} + type = object({ + amd_sev_snp = optional(string) + core_count = optional(number) + threads_per_core = optional(number) + }) + default = null } variable "credit_specification" { description = "Customize the credit specification of the instance" - type = map(string) - default = {} -} - -variable "elastic_gpu_specifications" { - description = "The elastic GPU to attach to the instance" - type = any - default = {} -} - -variable "elastic_inference_accelerator" { - description = "Configuration block containing an Elastic Inference Accelerator to attach to the instance" - type = map(string) - default = {} + type = object({ + cpu_credits = optional(string) + }) + default = null } variable "enclave_options" { description = "Enable Nitro Enclaves on launched instances" - type = map(string) - default = {} + type = object({ + enabled = optional(bool) + }) + default = null } variable "instance_market_options" { description = "The market (purchasing) option for the instance" - type = any - default = {} + type = object({ + market_type = optional(string) + spot_options = optional(object({ + block_duration_minutes = optional(number) + instance_interruption_behavior = optional(string) + max_price = optional(string) + spot_instance_type = optional(string) + valid_until = optional(string) + })) + }) + default = null } variable "maintenance_options" { description = "The maintenance options for the instance" - type = any - default = {} + type = object({ + auto_recovery = optional(string) + }) + default = null } variable "license_specifications" { - description = "A map of license specifications to associate with" - type = any - default = {} + description = "A list of license specifications to associate with" + type = list(object({ + license_configuration_arn = string + })) + default = null } variable "metadata_options" { description = "Customize the metadata options for the instance" - type = map(string) + type = object({ + http_endpoint = optional(string, "enabled") + http_protocol_ipv6 = optional(string) + http_put_response_hop_limit = optional(number, 1) + http_tokens = optional(string, "required") + instance_metadata_tags = optional(string) + }) default = { http_endpoint = "enabled" + http_put_response_hop_limit = 1 http_tokens = "required" - http_put_response_hop_limit = 2 } + nullable = false } -# TODO - make this false by default at next breaking change variable "enable_monitoring" { description = "Enables/disables detailed monitoring" type = bool - default = true + default = false + nullable = false } variable "enable_efa_support" { description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" type = bool default = false + nullable = false } -# TODO - make this true by default at next breaking change (remove variable, only pass indices) variable "enable_efa_only" { description = "Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later" type = bool - default = false + default = true + nullable = false } variable "efa_indices" { description = "The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true`" type = list(number) default = [0] + nullable = false } variable "network_interfaces" { description = "Customize network interfaces to be attached at instance boot time" - type = list(any) - default = [] + type = list(object({ + associate_carrier_ip_address = optional(bool) + associate_public_ip_address = optional(bool) + connection_tracking_specification = optional(object({ + tcp_established_timeout = optional(number) + udp_stream_timeout = optional(number) + udp_timeout = optional(number) + })) + delete_on_termination = optional(bool) + description = optional(string) + device_index = optional(number) + ena_srd_specification = optional(object({ + ena_srd_enabled = optional(bool) + ena_srd_udp_specification = optional(object({ + ena_srd_udp_enabled = optional(bool) + })) + })) + interface_type = optional(string) + ipv4_address_count = optional(number) + ipv4_addresses = optional(list(string)) + ipv4_prefix_count = optional(number) + ipv4_prefixes = optional(list(string)) + ipv6_address_count = optional(number) + ipv6_addresses = optional(list(string)) + ipv6_prefix_count = optional(number) + ipv6_prefixes = optional(list(string)) + network_card_index = optional(number) + network_interface_id = optional(string) + primary_ipv6 = optional(bool) + private_ip_address = optional(string) + security_groups = optional(list(string), []) + subnet_id = optional(string) + })) + default = [] + nullable = false } variable "placement" { description = "The placement of the instance" - type = map(string) - default = {} + type = object({ + affinity = optional(string) + availability_zone = optional(string) + group_name = optional(string) + host_id = optional(string) + host_resource_group_arn = optional(string) + partition_number = optional(number) + spread_domain = optional(string) + tenancy = optional(string) + }) + default = null } variable "create_placement_group" { description = "Determines whether a placement group is created & used by the node group" type = bool default = false -} - -# TODO - remove at next breaking change -variable "placement_group_strategy" { - description = "The placement group strategy" - type = string - default = "cluster" + nullable = false } variable "private_dns_name_options" { description = "The options for the instance hostname. The default values are inherited from the subnet" - type = map(string) - default = {} + type = object({ + enable_resource_name_dns_aaaa_record = optional(bool) + enable_resource_name_dns_a_record = optional(bool) + hostname_type = optional(string) + }) + default = null } variable "launch_template_tags" { @@ -340,6 +431,7 @@ variable "tag_specifications" { description = "The tags to apply to the resources during launch" type = list(string) default = ["instance", "volume", "network-interface"] + nullable = false } ################################################################################ @@ -352,28 +444,25 @@ variable "subnet_ids" { default = null } -variable "placement_group_az" { - description = "Availability zone where placement group is created (ex. `eu-west-1c`)" - type = string - default = null -} - variable "min_size" { description = "Minimum number of instances/nodes" type = number - default = 0 + default = 1 + nullable = false } variable "max_size" { description = "Maximum number of instances/nodes" type = number default = 3 + nullable = false } variable "desired_size" { description = "Desired number of instances/nodes" type = number default = 1 + nullable = false } variable "name" { @@ -386,12 +475,14 @@ variable "use_name_prefix" { description = "Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix" type = bool default = true + nullable = false } variable "ami_type" { description = "Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values" type = string - default = null + default = "AL2023_x86_64_STANDARD" + nullable = false } variable "ami_release_version" { @@ -401,15 +492,17 @@ variable "ami_release_version" { } variable "use_latest_ami_release_version" { - description = "Determines whether to use the latest AMI release version for the given `ami_type` (except for `CUSTOM`). Note: `ami_type` and `cluster_version` must be supplied in order to enable this feature" + description = "Determines whether to use the latest AMI release version for the given `ami_type` (except for `CUSTOM`). Note: `ami_type` and `kubernetes_version` must be supplied in order to enable this feature" type = bool - default = false + default = true + nullable = false } variable "capacity_type" { description = "Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT`" type = string default = "ON_DEMAND" + nullable = false } variable "disk_size" { @@ -436,7 +529,7 @@ variable "labels" { default = null } -variable "cluster_version" { +variable "kubernetes_version" { description = "Kubernetes version. Defaults to EKS Cluster Kubernetes version" type = string default = null @@ -450,22 +543,33 @@ variable "launch_template_version" { variable "remote_access" { description = "Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false`" - type = any - default = {} + type = object({ + ec2_ssh_key = optional(string) + source_security_group_ids = optional(list(string)) + }) + default = null } variable "taints" { description = "The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group" - type = any - default = {} + type = map(object({ + key = string + value = optional(string) + effect = string + })) + default = null } variable "update_config" { description = "Configuration block of settings for max unavailable resources during node group updates" - type = map(string) + type = object({ + max_unavailable = optional(number) + max_unavailable_percentage = optional(number) + }) default = { max_unavailable_percentage = 33 } + nullable = false } variable "node_repair_config" { @@ -478,8 +582,12 @@ variable "node_repair_config" { variable "timeouts" { description = "Create, update, and delete timeout configurations for the node group" - type = map(string) - default = {} + type = object({ + create = optional(string) + update = optional(string) + delete = optional(string) + }) + default = null } ################################################################################ @@ -490,12 +598,14 @@ variable "create_iam_role" { description = "Determines whether an IAM role is created or to use an existing IAM role" type = bool default = true + nullable = false } variable "cluster_ip_family" { description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" type = string default = "ipv4" + nullable = false } variable "iam_role_arn" { @@ -514,6 +624,7 @@ variable "iam_role_use_name_prefix" { description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" type = bool default = true + nullable = false } variable "iam_role_path" { @@ -525,7 +636,8 @@ variable "iam_role_path" { variable "iam_role_description" { description = "Description of the role" type = string - default = null + default = "EKS managed node group IAM role" + nullable = false } variable "iam_role_permissions_boundary" { @@ -538,18 +650,21 @@ variable "iam_role_attach_cni_policy" { description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" type = bool default = true + nullable = false } variable "iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" type = map(string) default = {} + nullable = false } variable "iam_role_tags" { description = "A map of additional tags to add to the IAM role created" type = map(string) default = {} + nullable = false } ################################################################################ @@ -560,26 +675,105 @@ variable "create_iam_role_policy" { description = "Determines whether an IAM role policy is created or not" type = bool default = true + nullable = false } variable "iam_role_policy_statements" { description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" - type = any - default = [] + type = list(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + })) + default = null } ################################################################################ -# Autoscaling Group Schedule +# Security Group ################################################################################ -variable "create_schedule" { - description = "Determines whether to create autoscaling group schedule or not" +variable "create_security_group" { + description = "Determines if a security group is created" type = bool default = true + nullable = false } -variable "schedules" { - description = "Map of autoscaling group schedule to create" - type = map(any) +variable "security_group_name" { + description = "Name to use on security group created" + type = string + default = null +} + +variable "security_group_use_name_prefix" { + description = "Determines whether the security group name (`security_group_name`) is used as a prefix" + type = bool + default = true + nullable = false +} + +variable "security_group_description" { + description = "Description of the security group created" + type = string + default = null +} + +variable "security_group_ingress_rules" { + description = "Security group ingress rules to add to the security group created" + type = map(object({ + name = optional(string) + + cidr_ipv4 = optional(string) + cidr_ipv6 = optional(string) + description = optional(string) + from_port = optional(string) + ip_protocol = optional(string, "tcp") + prefix_list_id = optional(string) + referenced_security_group_id = optional(string) + self = optional(bool, false) + tags = optional(map(string), {}) + to_port = optional(string) + })) + default = {} +} + +variable "security_group_egress_rules" { + description = "Security group egress rules to add to the security group created" + type = map(object({ + name = optional(string) + + cidr_ipv4 = optional(string) + cidr_ipv6 = optional(string) + description = optional(string) + from_port = optional(string) + ip_protocol = optional(string, "tcp") + prefix_list_id = optional(string) + referenced_security_group_id = optional(string) + self = optional(bool, false) + tags = optional(map(string), {}) + to_port = optional(string) + })) + default = {} +} + +variable "security_group_tags" { + description = "A map of additional tags to add to the security group created" + type = map(string) default = {} } diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 9b4cb99dd3..61ef2b980f 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -28,14 +28,14 @@ module "fargate_profile" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws](#provider\_aws) | >= 6.0 | ## Modules @@ -60,26 +60,29 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [account\_id](#input\_account\_id) | The AWS account ID - pass through value to reduce number of GET requests from data sources | `string` | `""` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | -| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `null` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | | [create](#input\_create) | Determines whether to create Fargate profile or not | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_iam\_role\_policy](#input\_create\_iam\_role\_policy) | Determines whether an IAM role policy is created or not | `bool` | `true` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the Fargate profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | -| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `"Fargate profile IAM role"` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `""` | no | | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | -| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed |
list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
}))
| `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [name](#input\_name) | Name of the EKS Fargate Profile | `string` | `""` | no | -| [selectors](#input\_selectors) | Configuration block(s) for selecting Kubernetes Pods to execute with this Fargate Profile | `any` | `[]` | no | +| [partition](#input\_partition) | The AWS partition - pass through value to reduce number of GET requests from data sources | `string` | `""` | no | +| [region](#input\_region) | Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration | `string` | `null` | no | +| [selectors](#input\_selectors) | Configuration block(s) for selecting Kubernetes Pods to execute with this Fargate Profile |
list(object({
labels = optional(map(string))
namespace = string
}))
| `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs for the EKS Fargate Profile | `list(string)` | `[]` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | -| [timeouts](#input\_timeouts) | Create and delete timeout configurations for the Fargate Profile | `map(string)` | `{}` | no | +| [timeouts](#input\_timeouts) | Create and delete timeout configurations for the Fargate Profile |
object({
create = optional(string)
delete = optional(string)
})
| `null` | no | ## Outputs diff --git a/modules/fargate-profile/main.tf b/modules/fargate-profile/main.tf index ffbbfb8294..78c94357f4 100644 --- a/modules/fargate-profile/main.tf +++ b/modules/fargate-profile/main.tf @@ -1,18 +1,32 @@ -data "aws_partition" "current" {} -data "aws_caller_identity" "current" {} -data "aws_region" "current" {} +data "aws_region" "current" { + count = var.create ? 1 : 0 + + region = var.region +} +data "aws_partition" "current" { + count = var.create && var.partition == "" ? 1 : 0 +} +data "aws_caller_identity" "current" { + count = var.create && var.account_id == "" ? 1 : 0 +} + +locals { + account_id = try(data.aws_caller_identity.current[0].account_id, var.account_id) + partition = try(data.aws_partition.current[0].partition, var.partition) + region = try(data.aws_region.current[0].region, "") +} locals { create_iam_role = var.create && var.create_iam_role iam_role_name = coalesce(var.iam_role_name, var.name, "fargate-profile") - iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" ipv4_cni_policy = { for k, v in { AmazonEKS_CNI_Policy = "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } ipv6_cni_policy = { for k, v in { - AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + AmazonEKS_CNI_IPv6_Policy = "arn:${local.partition}:iam::${local.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } } @@ -37,7 +51,7 @@ data "aws_iam_policy_document" "assume_role_policy" { variable = "aws:SourceArn" values = [ - "arn:${data.aws_partition.current.partition}:eks:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:fargateprofile/${var.cluster_name}/*", + "arn:${local.partition}:eks:${local.region}:${local.account_id}:fargateprofile/${var.cluster_name}/*", ] } } @@ -83,25 +97,25 @@ resource "aws_iam_role_policy_attachment" "additional" { ################################################################################ locals { - create_iam_role_policy = local.create_iam_role && var.create_iam_role_policy && length(var.iam_role_policy_statements) > 0 + create_iam_role_policy = local.create_iam_role && var.create_iam_role_policy && var.iam_role_policy_statements != null } data "aws_iam_policy_document" "role" { count = local.create_iam_role_policy ? 1 : 0 dynamic "statement" { - for_each = var.iam_role_policy_statements + for_each = var.iam_role_policy_statements != null ? var.iam_role_policy_statements : [] content { - sid = try(statement.value.sid, null) - actions = try(statement.value.actions, null) - not_actions = try(statement.value.not_actions, null) - effect = try(statement.value.effect, null) - resources = try(statement.value.resources, null) - not_resources = try(statement.value.not_resources, null) + sid = statement.value.sid + actions = statement.value.actions + not_actions = statement.value.not_actions + effect = statement.value.effect + resources = statement.value.resources + not_resources = statement.value.not_resources dynamic "principals" { - for_each = try(statement.value.principals, []) + for_each = statement.value.principals != null ? statement.value.principals : [] content { type = principals.value.type @@ -110,7 +124,7 @@ data "aws_iam_policy_document" "role" { } dynamic "not_principals" { - for_each = try(statement.value.not_principals, []) + for_each = statement.value.not_principals != null ? statement.value.not_principals : [] content { type = not_principals.value.type @@ -119,7 +133,7 @@ data "aws_iam_policy_document" "role" { } dynamic "condition" { - for_each = try(statement.value.conditions, []) + for_each = statement.value.condition != null ? statement.value.condition : [] content { test = condition.value.test @@ -147,25 +161,28 @@ resource "aws_iam_role_policy" "this" { resource "aws_eks_fargate_profile" "this" { count = var.create ? 1 : 0 + region = var.region + cluster_name = var.cluster_name fargate_profile_name = var.name pod_execution_role_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn subnet_ids = var.subnet_ids dynamic "selector" { - for_each = var.selectors + for_each = var.selectors != null ? var.selectors : [] content { namespace = selector.value.namespace - labels = lookup(selector.value, "labels", {}) + labels = selector.value.labels } } dynamic "timeouts" { - for_each = [var.timeouts] + for_each = var.timeouts != null ? [var.timeouts] : [] + content { - create = lookup(var.timeouts, "create", null) - delete = lookup(var.timeouts, "delete", null) + create = var.timeouts.create + delete = var.timeouts.delete } } diff --git a/modules/fargate-profile/variables.tf b/modules/fargate-profile/variables.tf index 3e37b8c151..5d87e56447 100644 --- a/modules/fargate-profile/variables.tf +++ b/modules/fargate-profile/variables.tf @@ -2,12 +2,32 @@ variable "create" { description = "Determines whether to create Fargate profile or not" type = bool default = true + nullable = false } variable "tags" { description = "A map of tags to add to all resources" type = map(string) default = {} + nullable = false +} + +variable "region" { + description = "Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration" + type = string + default = null +} + +variable "partition" { + description = "The AWS partition - pass through value to reduce number of GET requests from data sources" + type = string + default = "" +} + +variable "account_id" { + description = "The AWS account ID - pass through value to reduce number of GET requests from data sources" + type = string + default = "" } ################################################################################ @@ -18,6 +38,7 @@ variable "create_iam_role" { description = "Determines whether an IAM role is created or to use an existing IAM role" type = bool default = true + nullable = false } variable "cluster_ip_family" { @@ -42,6 +63,7 @@ variable "iam_role_use_name_prefix" { description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" type = bool default = true + nullable = false } variable "iam_role_path" { @@ -53,7 +75,8 @@ variable "iam_role_path" { variable "iam_role_description" { description = "Description of the role" type = string - default = null + default = "Fargate profile IAM role" + nullable = false } variable "iam_role_permissions_boundary" { @@ -66,18 +89,21 @@ variable "iam_role_attach_cni_policy" { description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" type = bool default = true + nullable = false } variable "iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" type = map(string) default = {} + nullable = false } variable "iam_role_tags" { description = "A map of additional tags to add to the IAM role created" type = map(string) default = {} + nullable = false } ################################################################################ @@ -88,12 +114,33 @@ variable "create_iam_role_policy" { description = "Determines whether an IAM role policy is created or not" type = bool default = true + nullable = false } variable "iam_role_policy_statements" { description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" - type = any - default = [] + type = list(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + })) + default = null } ################################################################################ @@ -103,29 +150,37 @@ variable "iam_role_policy_statements" { variable "cluster_name" { description = "Name of the EKS cluster" type = string - default = null + default = "" } variable "name" { description = "Name of the EKS Fargate Profile" type = string default = "" + nullable = false } variable "subnet_ids" { description = "A list of subnet IDs for the EKS Fargate Profile" type = list(string) default = [] + nullable = false } variable "selectors" { description = "Configuration block(s) for selecting Kubernetes Pods to execute with this Fargate Profile" - type = any - default = [] + type = list(object({ + labels = optional(map(string)) + namespace = string + })) + default = null } variable "timeouts" { description = "Create and delete timeout configurations for the Fargate Profile" - type = map(string) - default = {} + type = object({ + create = optional(string) + delete = optional(string) + }) + default = null } diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index 3119747f76..176aabcdf4 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -74,14 +74,14 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws](#provider\_aws) | >= 6.0 | ## Modules @@ -115,7 +115,7 @@ No modules. | [enable\_ira](#input\_enable\_ira) | Enables IAM Roles Anywhere based IAM permissions on the node | `bool` | `false` | no | | [enable\_pod\_identity](#input\_enable\_pod\_identity) | Enables EKS Pod Identity based IAM permissions on the node | `bool` | `true` | no | | [intermediate\_policy\_name](#input\_intermediate\_policy\_name) | Name of the IAM policy | `string` | `null` | no | -| [intermediate\_policy\_statements](#input\_intermediate\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [intermediate\_policy\_statements](#input\_intermediate\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed |
list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
}))
| `null` | no | | [intermediate\_policy\_use\_name\_prefix](#input\_intermediate\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`intermediate_policy_name`) is used as a prefix | `bool` | `true` | no | | [intermediate\_role\_description](#input\_intermediate\_role\_description) | IAM role description | `string` | `"EKS Hybrid Node IAM Roles Anywhere intermediate IAM role"` | no | | [intermediate\_role\_name](#input\_intermediate\_role\_name) | Name of the IAM role | `string` | `null` | no | @@ -129,7 +129,7 @@ No modules. | [ira\_profile\_session\_policy](#input\_ira\_profile\_session\_policy) | A session policy that applies to the trust boundary of the vended session credentials | `string` | `null` | no | | [ira\_trust\_anchor\_acm\_pca\_arn](#input\_ira\_trust\_anchor\_acm\_pca\_arn) | The ARN of the ACM PCA that issued the trust anchor certificate | `string` | `null` | no | | [ira\_trust\_anchor\_name](#input\_ira\_trust\_anchor\_name) | Name of the Roles Anywhere trust anchor | `string` | `null` | no | -| [ira\_trust\_anchor\_notification\_settings](#input\_ira\_trust\_anchor\_notification\_settings) | Notification settings for the trust anchor | `any` | `[]` | no | +| [ira\_trust\_anchor\_notification\_settings](#input\_ira\_trust\_anchor\_notification\_settings) | Notification settings for the trust anchor |
list(object({
channel = optional(string)
enabled = optional(bool)
event = optional(string)
threshold = optional(number)
}))
| `null` | no | | [ira\_trust\_anchor\_source\_type](#input\_ira\_trust\_anchor\_source\_type) | The source type of the trust anchor | `string` | `null` | no | | [ira\_trust\_anchor\_x509\_certificate\_data](#input\_ira\_trust\_anchor\_x509\_certificate\_data) | The X.509 certificate data of the trust anchor | `string` | `null` | no | | [max\_session\_duration](#input\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | @@ -140,9 +140,9 @@ No modules. | [policy\_description](#input\_policy\_description) | IAM policy description | `string` | `"EKS Hybrid Node IAM role policy"` | no | | [policy\_name](#input\_policy\_name) | Name of the IAM policy | `string` | `"EKSHybridNode"` | no | | [policy\_path](#input\_policy\_path) | Path of the IAM policy | `string` | `"/"` | no | -| [policy\_statements](#input\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [policy\_statements](#input\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed |
list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
}))
| `null` | no | | [policy\_use\_name\_prefix](#input\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`policy_name`) is used as a prefix | `bool` | `true` | no | -| [tags](#input\_tags) | A map of additional tags to add the the IAM role | `map(any)` | `{}` | no | +| [tags](#input\_tags) | A map of additional tags to add the the IAM role | `map(string)` | `{}` | no | | [trust\_anchor\_arns](#input\_trust\_anchor\_arns) | List of IAM Roles Anywhere trust anchor ARNs. Required if `enable_ira` is set to `true` | `list(string)` | `[]` | no | | [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether the name of the IAM role (`name`) is used as a prefix | `bool` | `true` | no | diff --git a/modules/hybrid-node-role/main.tf b/modules/hybrid-node-role/main.tf index 5a2473f29b..1d14f41b40 100644 --- a/modules/hybrid-node-role/main.tf +++ b/modules/hybrid-node-role/main.tf @@ -3,7 +3,7 @@ data "aws_partition" "current" { } locals { - partition = try(data.aws_partition.current[0].partition, "aws") + partition = try(data.aws_partition.current[0].partition, "") } ################################################################################ @@ -117,18 +117,18 @@ data "aws_iam_policy_document" "this" { } dynamic "statement" { - for_each = var.policy_statements + for_each = var.policy_statements != null ? var.policy_statements : [] content { - sid = try(statement.value.sid, null) - actions = try(statement.value.actions, null) - not_actions = try(statement.value.not_actions, null) - effect = try(statement.value.effect, null) - resources = try(statement.value.resources, null) - not_resources = try(statement.value.not_resources, null) + sid = statement.value.sid + actions = statement.value.actions + not_actions = statement.value.not_actions + effect = statement.value.effect + resources = statement.value.resources + not_resources = statement.value.not_resources dynamic "principals" { - for_each = try(statement.value.principals, []) + for_each = statement.value.principals != null ? statement.value.principals : [] content { type = principals.value.type @@ -137,7 +137,7 @@ data "aws_iam_policy_document" "this" { } dynamic "not_principals" { - for_each = try(statement.value.not_principals, []) + for_each = statement.value.not_principals != null ? statement.value.not_principals : [] content { type = not_principals.value.type @@ -146,7 +146,7 @@ data "aws_iam_policy_document" "this" { } dynamic "condition" { - for_each = try(statement.value.conditions, []) + for_each = statement.value.condition != null ? statement.value.condition : [] content { test = condition.value.test @@ -215,7 +215,7 @@ resource "aws_rolesanywhere_trust_anchor" "this" { name = try(coalesce(var.ira_trust_anchor_name, var.name), null) dynamic "notification_settings" { - for_each = var.ira_trust_anchor_notification_settings + for_each = var.ira_trust_anchor_notification_settings != null ? var.ira_trust_anchor_notification_settings : [] content { channel = try(notification_settings.value.channel, null) @@ -297,18 +297,18 @@ data "aws_iam_policy_document" "intermediate" { } dynamic "statement" { - for_each = var.intermediate_policy_statements + for_each = var.intermediate_policy_statements != null ? var.intermediate_policy_statements : [] content { - sid = try(statement.value.sid, null) - actions = try(statement.value.actions, null) - not_actions = try(statement.value.not_actions, null) - effect = try(statement.value.effect, null) - resources = try(statement.value.resources, null) - not_resources = try(statement.value.not_resources, null) + sid = statement.value.sid + actions = statement.value.actions + not_actions = statement.value.not_actions + effect = statement.value.effect + resources = statement.value.resources + not_resources = statement.value.not_resources dynamic "principals" { - for_each = try(statement.value.principals, []) + for_each = statement.value.principals != null ? statement.value.principals : [] content { type = principals.value.type @@ -317,7 +317,7 @@ data "aws_iam_policy_document" "intermediate" { } dynamic "not_principals" { - for_each = try(statement.value.not_principals, []) + for_each = statement.value.not_principals != null ? statement.value.not_principals : [] content { type = not_principals.value.type @@ -326,7 +326,7 @@ data "aws_iam_policy_document" "intermediate" { } dynamic "condition" { - for_each = try(statement.value.conditions, []) + for_each = statement.value.condition != null ? statement.value.condition : [] content { test = condition.value.test diff --git a/modules/hybrid-node-role/variables.tf b/modules/hybrid-node-role/variables.tf index 6e8d80465e..34e1247667 100644 --- a/modules/hybrid-node-role/variables.tf +++ b/modules/hybrid-node-role/variables.tf @@ -46,7 +46,7 @@ variable "permissions_boundary_arn" { variable "tags" { description = "A map of additional tags to add the the IAM role" - type = map(any) + type = map(string) default = {} } @@ -92,8 +92,28 @@ variable "policy_description" { variable "policy_statements" { description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" - type = any - default = [] + type = list(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + })) + default = null } variable "policies" { @@ -160,8 +180,13 @@ variable "ira_trust_anchor_name" { variable "ira_trust_anchor_notification_settings" { description = "Notification settings for the trust anchor" - type = any - default = [] + type = list(object({ + channel = optional(string) + enabled = optional(bool) + event = optional(string) + threshold = optional(number) + })) + default = null } variable "ira_trust_anchor_acm_pca_arn" { @@ -228,8 +253,28 @@ variable "intermediate_policy_use_name_prefix" { variable "intermediate_policy_statements" { description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" - type = any - default = [] + type = list(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + })) + default = null } variable "intermediate_role_policies" { diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 43cc3b560b..85e472b225 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -85,14 +85,14 @@ module "karpenter" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws](#provider\_aws) | >= 6.0 | ## Modules @@ -121,8 +121,6 @@ No modules. | [aws_iam_policy_document.controller_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.node_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.v033](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.v1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | @@ -139,27 +137,23 @@ No modules. | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created | `bool` | `true` | no | | [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `false` | no | | [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | -| [create\_pod\_identity\_association](#input\_create\_pod\_identity\_association) | Determines whether to create pod identity association | `bool` | `false` | no | -| [enable\_irsa](#input\_enable\_irsa) | Determines whether to enable support for IAM role for service accounts | `bool` | `false` | no | -| [enable\_pod\_identity](#input\_enable\_pod\_identity) | Determines whether to enable support for EKS pod identity | `bool` | `true` | no | +| [create\_pod\_identity\_association](#input\_create\_pod\_identity\_association) | Determines whether to create pod identity association | `bool` | `true` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | -| [enable\_v1\_permissions](#input\_enable\_v1\_permissions) | Determines whether to enable permissions suitable for v1+ (`true`) or for v0.33.x-v0.37.x (`false`) | `bool` | `false` | no | | [iam\_policy\_description](#input\_iam\_policy\_description) | IAM policy description | `string` | `"Karpenter controller IAM policy"` | no | | [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy | `string` | `"KarpenterController"` | no | | [iam\_policy\_path](#input\_iam\_policy\_path) | Path of the IAM policy | `string` | `"/"` | no | -| [iam\_policy\_statements](#input\_iam\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [iam\_policy\_statements](#input\_iam\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed |
list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
}))
| `null` | no | | [iam\_policy\_use\_name\_prefix](#input\_iam\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`iam_policy_name`) is used as a prefix | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | IAM role description | `string` | `"Karpenter controller IAM role"` | no | | [iam\_role\_max\_session\_duration](#input\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name of the IAM role | `string` | `"KarpenterController"` | no | +| [iam\_role\_override\_assume\_policy\_documents](#input\_iam\_role\_override\_assume\_policy\_documents) | A list of IAM policy documents to override the default assume role policy document for the Karpenter controller IAM role | `list(string)` | `[]` | no | | [iam\_role\_path](#input\_iam\_role\_path) | Path of the IAM role | `string` | `"/"` | no | | [iam\_role\_permissions\_boundary\_arn](#input\_iam\_role\_permissions\_boundary\_arn) | Permissions boundary ARN to use for the IAM role | `string` | `null` | no | | [iam\_role\_policies](#input\_iam\_role\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | -| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add the the IAM role | `map(any)` | `{}` | no | +| [iam\_role\_source\_assume\_policy\_documents](#input\_iam\_role\_source\_assume\_policy\_documents) | A list of IAM policy documents to use as a source for the assume role policy document for the Karpenter controller IAM role | `list(string)` | `[]` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add the the IAM role | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the name of the IAM role (`iam_role_name`) is used as a prefix | `bool` | `true` | no | -| [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no | -| [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` |
[
"karpenter:karpenter"
]
| no | -| [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | | [namespace](#input\_namespace) | Namespace to associate with the Karpenter Pod Identity | `string` | `"kube-system"` | no | | [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [node\_iam\_role\_arn](#input\_node\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | @@ -175,6 +169,7 @@ No modules. | [queue\_kms\_master\_key\_id](#input\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no | | [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | | [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no | +| [region](#input\_region) | Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration | `string` | `null` | no | | [rule\_name\_prefix](#input\_rule\_name\_prefix) | Prefix used for all event bridge rules | `string` | `"Karpenter"` | no | | [service\_account](#input\_service\_account) | Service account to associate with the Karpenter Pod Identity | `string` | `"karpenter"` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index d03dfa49f8..7eba2cb5d8 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -1,12 +1,20 @@ -data "aws_region" "current" {} -data "aws_partition" "current" {} -data "aws_caller_identity" "current" {} +data "aws_region" "current" { + count = var.create ? 1 : 0 + + region = var.region +} +data "aws_partition" "current" { + count = var.create ? 1 : 0 +} +data "aws_caller_identity" "current" { + count = var.create ? 1 : 0 +} locals { - account_id = data.aws_caller_identity.current.account_id - dns_suffix = data.aws_partition.current.dns_suffix - partition = data.aws_partition.current.partition - region = data.aws_region.current.name + account_id = try(data.aws_caller_identity.current[0].account_id, "") + dns_suffix = try(data.aws_partition.current[0].dns_suffix, "") + partition = try(data.aws_partition.current[0].partition, "") + region = try(data.aws_region.current[0].region, "") } ################################################################################ @@ -14,54 +22,26 @@ locals { ################################################################################ locals { - create_iam_role = var.create && var.create_iam_role - irsa_oidc_provider_url = replace(var.irsa_oidc_provider_arn, "/^(.*provider/)/", "") + create_iam_role = var.create && var.create_iam_role } data "aws_iam_policy_document" "controller_assume_role" { count = local.create_iam_role ? 1 : 0 - # Pod Identity - dynamic "statement" { - for_each = var.enable_pod_identity ? [1] : [] + override_policy_documents = var.iam_role_override_assume_policy_documents + source_policy_documents = var.iam_role_source_assume_policy_documents - content { - actions = [ - "sts:AssumeRole", - "sts:TagSession", - ] - - principals { - type = "Service" - identifiers = ["pods.eks.amazonaws.com"] - } - } - } - - # IAM Roles for Service Accounts (IRSA) - dynamic "statement" { - for_each = var.enable_irsa ? [1] : [] - - content { - actions = ["sts:AssumeRoleWithWebIdentity"] - - principals { - type = "Federated" - identifiers = [var.irsa_oidc_provider_arn] - } - - condition { - test = var.irsa_assume_role_condition_test - variable = "${local.irsa_oidc_provider_url}:sub" - values = [for sa in var.irsa_namespace_service_accounts : "system:serviceaccount:${sa}"] - } + # Pod Identity + statement { + sid = "PodIdentity" + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] - # https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-oidc-and-irsa/?nc1=h_ls - condition { - test = var.irsa_assume_role_condition_test - variable = "${local.irsa_oidc_provider_url}:aud" - values = ["sts.amazonaws.com"] - } + principals { + type = "Service" + identifiers = ["pods.eks.amazonaws.com"] } } } @@ -82,12 +62,6 @@ resource "aws_iam_role" "controller" { tags = merge(var.tags, var.iam_role_tags) } -data "aws_iam_policy_document" "controller" { - count = local.create_iam_role ? 1 : 0 - - source_policy_documents = var.enable_v1_permissions ? [data.aws_iam_policy_document.v1[0].json] : [data.aws_iam_policy_document.v033[0].json] -} - resource "aws_iam_policy" "controller" { count = local.create_iam_role ? 1 : 0 @@ -119,7 +93,9 @@ resource "aws_iam_role_policy_attachment" "controller_additional" { ################################################################################ resource "aws_eks_pod_identity_association" "karpenter" { - count = local.create_iam_role && var.enable_pod_identity && var.create_pod_identity_association ? 1 : 0 + count = local.create_iam_role && var.create_pod_identity_association ? 1 : 0 + + region = var.region cluster_name = var.cluster_name namespace = var.namespace @@ -142,6 +118,8 @@ locals { resource "aws_sqs_queue" "this" { count = local.enable_spot_termination ? 1 : 0 + region = var.region + name = local.queue_name message_retention_seconds = 300 sqs_managed_sse_enabled = var.queue_managed_sse_enabled ? var.queue_managed_sse_enabled : null @@ -175,7 +153,7 @@ data "aws_iam_policy_document" "queue" { ] resources = [aws_sqs_queue.this[0].arn] condition { - test = "StringEquals" + test = "Bool" variable = "aws:SecureTransport" values = [ "false" @@ -193,6 +171,8 @@ data "aws_iam_policy_document" "queue" { resource "aws_sqs_queue_policy" "this" { count = local.enable_spot_termination ? 1 : 0 + region = var.region + queue_url = aws_sqs_queue.this[0].url policy = data.aws_iam_policy_document.queue[0].json } @@ -241,6 +221,8 @@ locals { resource "aws_cloudwatch_event_rule" "this" { for_each = { for k, v in local.events : k => v if local.enable_spot_termination } + region = var.region + name_prefix = "${var.rule_name_prefix}${each.value.name}-" description = each.value.description event_pattern = jsonencode(each.value.event_pattern) @@ -254,6 +236,8 @@ resource "aws_cloudwatch_event_rule" "this" { resource "aws_cloudwatch_event_target" "this" { for_each = { for k, v in local.events : k => v if local.enable_spot_termination } + region = var.region + rule = aws_cloudwatch_event_rule.this[each.key].name target_id = "KarpenterInterruptionQueueTarget" arn = aws_sqs_queue.this[0].arn @@ -274,7 +258,7 @@ locals { AmazonEKS_CNI_Policy = "${local.node_iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } : k => v if var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } ipv6_cni_policy = { for k, v in { - AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + AmazonEKS_CNI_IPv6_Policy = "arn:${local.partition}:iam::${local.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" } : k => v if var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } } @@ -337,6 +321,8 @@ resource "aws_iam_role_policy_attachment" "node_additional" { resource "aws_eks_access_entry" "node" { count = var.create && var.create_access_entry ? 1 : 0 + region = var.region + cluster_name = var.cluster_name principal_arn = var.create_node_iam_role ? aws_iam_role.node[0].arn : var.node_iam_role_arn type = var.access_entry_type diff --git a/modules/karpenter/policy.tf b/modules/karpenter/policy.tf index b9068d7b60..16f28fa61d 100644 --- a/modules/karpenter/policy.tf +++ b/modules/karpenter/policy.tf @@ -1,357 +1,4 @@ -################################################################################ -# v0.33.x - v0.37.x Controller IAM Policy -################################################################################ - -data "aws_iam_policy_document" "v033" { - count = local.create_iam_role ? 1 : 0 - - statement { - sid = "AllowScopedEC2InstanceActions" - resources = [ - "arn:${local.partition}:ec2:*::image/*", - "arn:${local.partition}:ec2:*::snapshot/*", - "arn:${local.partition}:ec2:*:*:spot-instances-request/*", - "arn:${local.partition}:ec2:*:*:security-group/*", - "arn:${local.partition}:ec2:*:*:subnet/*", - "arn:${local.partition}:ec2:*:*:launch-template/*", - ] - - actions = [ - "ec2:RunInstances", - "ec2:CreateFleet" - ] - } - - statement { - sid = "AllowScopedEC2InstanceActionsWithTags" - resources = [ - "arn:${local.partition}:ec2:*:*:fleet/*", - "arn:${local.partition}:ec2:*:*:instance/*", - "arn:${local.partition}:ec2:*:*:volume/*", - "arn:${local.partition}:ec2:*:*:network-interface/*", - "arn:${local.partition}:ec2:*:*:launch-template/*", - "arn:${local.partition}:ec2:*:*:spot-instances-request/*", - ] - actions = [ - "ec2:RunInstances", - "ec2:CreateFleet", - "ec2:CreateLaunchTemplate" - ] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringLike" - variable = "aws:RequestTag/karpenter.sh/nodepool" - values = ["*"] - } - } - - statement { - sid = "AllowScopedResourceCreationTagging" - resources = [ - "arn:${local.partition}:ec2:*:*:fleet/*", - "arn:${local.partition}:ec2:*:*:instance/*", - "arn:${local.partition}:ec2:*:*:volume/*", - "arn:${local.partition}:ec2:*:*:network-interface/*", - "arn:${local.partition}:ec2:*:*:launch-template/*", - "arn:${local.partition}:ec2:*:*:spot-instances-request/*", - ] - actions = ["ec2:CreateTags"] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "ec2:CreateAction" - values = [ - "RunInstances", - "CreateFleet", - "CreateLaunchTemplate", - ] - } - - condition { - test = "StringLike" - variable = "aws:RequestTag/karpenter.sh/nodepool" - values = ["*"] - } - } - - statement { - sid = "AllowScopedResourceTagging" - resources = ["arn:${local.partition}:ec2:*:*:instance/*"] - actions = ["ec2:CreateTags"] - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringLike" - variable = "aws:ResourceTag/karpenter.sh/nodepool" - values = ["*"] - } - - condition { - test = "ForAllValues:StringEquals" - variable = "aws:TagKeys" - values = [ - "karpenter.sh/nodeclaim", - "Name", - ] - } - } - - statement { - sid = "AllowScopedDeletion" - resources = [ - "arn:${local.partition}:ec2:*:*:instance/*", - "arn:${local.partition}:ec2:*:*:launch-template/*" - ] - - actions = [ - "ec2:TerminateInstances", - "ec2:DeleteLaunchTemplate" - ] - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringLike" - variable = "aws:ResourceTag/karpenter.sh/nodepool" - values = ["*"] - } - } - - statement { - sid = "AllowRegionalReadActions" - resources = ["*"] - actions = [ - "ec2:DescribeAvailabilityZones", - "ec2:DescribeImages", - "ec2:DescribeInstances", - "ec2:DescribeInstanceTypeOfferings", - "ec2:DescribeInstanceTypes", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSpotPriceHistory", - "ec2:DescribeSubnets" - ] - - condition { - test = "StringEquals" - variable = "aws:RequestedRegion" - values = [local.region] - } - } - - statement { - sid = "AllowSSMReadActions" - resources = coalescelist(var.ami_id_ssm_parameter_arns, ["arn:${local.partition}:ssm:${local.region}::parameter/aws/service/*"]) - actions = ["ssm:GetParameter"] - } - - statement { - sid = "AllowPricingReadActions" - resources = ["*"] - actions = ["pricing:GetProducts"] - } - - dynamic "statement" { - for_each = local.enable_spot_termination ? [1] : [] - - content { - sid = "AllowInterruptionQueueActions" - resources = [try(aws_sqs_queue.this[0].arn, null)] - actions = [ - "sqs:DeleteMessage", - "sqs:GetQueueAttributes", - "sqs:GetQueueUrl", - "sqs:ReceiveMessage" - ] - } - } - - statement { - sid = "AllowPassingInstanceRole" - resources = var.create_node_iam_role ? [aws_iam_role.node[0].arn] : [var.node_iam_role_arn] - actions = ["iam:PassRole"] - - condition { - test = "StringEquals" - variable = "iam:PassedToService" - values = ["ec2.${local.dns_suffix}"] - } - } - - statement { - sid = "AllowScopedInstanceProfileCreationActions" - resources = ["*"] - actions = ["iam:CreateInstanceProfile"] - - condition { - test = "StringEquals" - variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "aws:RequestTag/topology.kubernetes.io/region" - values = [local.region] - } - - condition { - test = "StringLike" - variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" - values = ["*"] - } - } - - statement { - sid = "AllowScopedInstanceProfileTagActions" - resources = ["*"] - actions = ["iam:TagInstanceProfile"] - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/topology.kubernetes.io/region" - values = [local.region] - } - - condition { - test = "StringEquals" - variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/topology.kubernetes.io/region" - values = [local.region] - } - - condition { - test = "StringLike" - variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" - values = ["*"] - } - - condition { - test = "StringLike" - variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" - values = ["*"] - } - } - - statement { - sid = "AllowScopedInstanceProfileActions" - resources = ["*"] - actions = [ - "iam:AddRoleToInstanceProfile", - "iam:RemoveRoleFromInstanceProfile", - "iam:DeleteInstanceProfile" - ] - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" - values = ["owned"] - } - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/topology.kubernetes.io/region" - values = [local.region] - } - - condition { - test = "StringLike" - variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" - values = ["*"] - } - } - - statement { - sid = "AllowInstanceProfileReadActions" - resources = ["*"] - actions = ["iam:GetInstanceProfile"] - } - - statement { - sid = "AllowAPIServerEndpointDiscovery" - resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] - actions = ["eks:DescribeCluster"] - } - - dynamic "statement" { - for_each = var.iam_policy_statements - - content { - sid = try(statement.value.sid, null) - actions = try(statement.value.actions, null) - not_actions = try(statement.value.not_actions, null) - effect = try(statement.value.effect, null) - resources = try(statement.value.resources, null) - not_resources = try(statement.value.not_resources, null) - - dynamic "principals" { - for_each = try(statement.value.principals, []) - - content { - type = principals.value.type - identifiers = principals.value.identifiers - } - } - - dynamic "not_principals" { - for_each = try(statement.value.not_principals, []) - - content { - type = not_principals.value.type - identifiers = not_principals.value.identifiers - } - } - - dynamic "condition" { - for_each = try(statement.value.conditions, []) - - content { - test = condition.value.test - values = condition.value.values - variable = condition.value.variable - } - } - } - } -} - -################################################################################ -# v1.0.x Controller IAM Policy -################################################################################ - -data "aws_iam_policy_document" "v1" { +data "aws_iam_policy_document" "controller" { count = local.create_iam_role ? 1 : 0 statement { @@ -708,18 +355,18 @@ data "aws_iam_policy_document" "v1" { } dynamic "statement" { - for_each = var.iam_policy_statements + for_each = var.iam_policy_statements != null ? var.iam_policy_statements : [] content { - sid = try(statement.value.sid, null) - actions = try(statement.value.actions, null) - not_actions = try(statement.value.not_actions, null) - effect = try(statement.value.effect, null) - resources = try(statement.value.resources, null) - not_resources = try(statement.value.not_resources, null) + sid = statement.value.sid + actions = statement.value.actions + not_actions = statement.value.not_actions + effect = statement.value.effect + resources = statement.value.resources + not_resources = statement.value.not_resources dynamic "principals" { - for_each = try(statement.value.principals, []) + for_each = statement.value.principals != null ? statement.value.principals : [] content { type = principals.value.type @@ -728,7 +375,7 @@ data "aws_iam_policy_document" "v1" { } dynamic "not_principals" { - for_each = try(statement.value.not_principals, []) + for_each = statement.value.not_principals != null ? statement.value.not_principals : [] content { type = not_principals.value.type @@ -737,7 +384,7 @@ data "aws_iam_policy_document" "v1" { } dynamic "condition" { - for_each = try(statement.value.conditions, []) + for_each = statement.value.condition != null ? statement.value.condition : [] content { test = condition.value.test diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 71b2cbdf38..f0725f6a58 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -16,6 +16,12 @@ variable "cluster_name" { default = "" } +variable "region" { + description = "Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration" + type = string + default = null +} + ################################################################################ # Karpenter controller IAM Role ################################################################################ @@ -64,7 +70,7 @@ variable "iam_role_permissions_boundary_arn" { variable "iam_role_tags" { description = "A map of additional tags to add the the IAM role" - type = map(any) + type = map(string) default = {} } @@ -92,10 +98,42 @@ variable "iam_policy_description" { default = "Karpenter controller IAM policy" } +variable "iam_role_override_assume_policy_documents" { + description = "A list of IAM policy documents to override the default assume role policy document for the Karpenter controller IAM role" + type = list(string) + default = [] +} + +variable "iam_role_source_assume_policy_documents" { + description = "A list of IAM policy documents to use as a source for the assume role policy document for the Karpenter controller IAM role" + type = list(string) + default = [] +} + variable "iam_policy_statements" { description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" - type = any - default = [] + type = list(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + })) + default = null } variable "iam_role_policies" { @@ -110,55 +148,14 @@ variable "ami_id_ssm_parameter_arns" { default = [] } -variable "enable_pod_identity" { - description = "Determines whether to enable support for EKS pod identity" - type = bool - default = true -} - -# TODO - make v1 permssions the default policy at next breaking change -variable "enable_v1_permissions" { - description = "Determines whether to enable permissions suitable for v1+ (`true`) or for v0.33.x-v0.37.x (`false`)" - type = bool - default = false -} - -################################################################################ -# IAM Role for Service Account (IRSA) -################################################################################ - -variable "enable_irsa" { - description = "Determines whether to enable support for IAM role for service accounts" - type = bool - default = false -} - -variable "irsa_oidc_provider_arn" { - description = "OIDC provider arn used in trust policy for IAM role for service accounts" - type = string - default = "" -} - -variable "irsa_namespace_service_accounts" { - description = "List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts" - type = list(string) - default = ["karpenter:karpenter"] -} - -variable "irsa_assume_role_condition_test" { - description = "Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role" - type = string - default = "StringEquals" -} - ################################################################################ # Pod Identity Association ################################################################################ -# TODO - Change default to `true` at next breaking change + variable "create_pod_identity_association" { description = "Determines whether to create pod identity association" type = bool - default = false + default = true } variable "namespace" { diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 06747514f2..8b21f20b69 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -10,7 +10,7 @@ module "self_managed_node_group" { name = "separate-self-mng" cluster_name = "my-cluster" - cluster_version = "1.31" + kubernetes_version = "1.31" cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" @@ -42,14 +42,14 @@ module "self_managed_node_group" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws](#provider\_aws) | >= 6.0 | ## Modules @@ -62,7 +62,6 @@ module "self_managed_node_group" { | Name | Type | |------|------| | [aws_autoscaling_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | -| [aws_autoscaling_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_schedule) | resource | | [aws_eks_access_entry.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | | [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | @@ -71,39 +70,41 @@ module "self_managed_node_group" { | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | | [aws_placement_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/placement_group) | resource | +| [aws_security_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_vpc_security_group_egress_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_egress_rule) | resource | +| [aws_vpc_security_group_ingress_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | -| [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_ssm_parameter.ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | -| [aws_subnets.placement_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | +| [aws_subnet.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [additional\_cluster\_dns\_ips](#input\_additional\_cluster\_dns\_ips) | Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*` | `list(string)` | `[]` | no | +| [account\_id](#input\_account\_id) | The AWS account ID - pass through value to reduce number of GET requests from data sources | `string` | `""` | no | +| [additional\_cluster\_dns\_ips](#input\_additional\_cluster\_dns\_ips) | Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*` | `list(string)` | `null` | no | | [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance | `string` | `""` | no | -| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the node group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `"AL2_x86_64"` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the node group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `"AL2023_x86_64_STANDARD"` | no | | [autoscaling\_group\_tags](#input\_autoscaling\_group\_tags) | A map of additional tags to add to the autoscaling group created. Tags are applied to the autoscaling group only and are NOT propagated to instances | `map(string)` | `{}` | no | | [availability\_zones](#input\_availability\_zones) | A list of one or more availability zones for the group. Used for EC2-Classic and default subnets when not specified with `subnet_ids` argument. Conflicts with `subnet_ids` | `list(string)` | `null` | no | -| [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | -| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI |
map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
}))
| `null` | no | +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `null` | no | | [capacity\_rebalance](#input\_capacity\_rebalance) | Indicates whether capacity rebalance is enabled | `bool` | `null` | no | -| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | -| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | -| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | -| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | -| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | -| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | +| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations |
object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
})
| `null` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `null` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `null` | no | +| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `null` | no | +| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `null` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `""` | no | | [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | -| [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | +| [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `null` | no | | [context](#input\_context) | Reserved | `string` | `null` | no | -| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | +| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance |
object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
})
| `null` | no | | [create](#input\_create) | Determines whether to create self managed node group or not | `bool` | `true` | no | | [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the node group | `bool` | `true` | no | | [create\_autoscaling\_group](#input\_create\_autoscaling\_group) | Determines whether to create autoscaling group or not | `bool` | `true` | no | @@ -111,49 +112,44 @@ module "self_managed_node_group" { | [create\_iam\_role\_policy](#input\_create\_iam\_role\_policy) | Determines whether an IAM role policy is created or not | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create launch template or not | `bool` | `true` | no | | [create\_placement\_group](#input\_create\_placement\_group) | Determines whether a placement group is created & used by the node group | `bool` | `false` | no | -| [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | -| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | -| [default\_cooldown](#input\_default\_cooldown) | The amount of time, in seconds, after a scaling activity completes before another scaling activity can start | `number` | `null` | no | +| [create\_security\_group](#input\_create\_security\_group) | Determines if a security group is created | `bool` | `true` | no | +| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance |
object({
cpu_credits = optional(string)
})
| `null` | no | | [default\_instance\_warmup](#input\_default\_instance\_warmup) | Amount of time, in seconds, until a newly launched instance can contribute to the Amazon CloudWatch metrics. This delay lets an instance finish initializing before Amazon EC2 Auto Scaling aggregates instance metrics, resulting in more reliable usage data | `number` | `null` | no | -| [delete\_timeout](#input\_delete\_timeout) | Delete timeout to wait for destroying autoscaling group | `string` | `null` | no | | [desired\_size](#input\_desired\_size) | The number of Amazon EC2 instances that should be running in the autoscaling group | `number` | `1` | no | | [desired\_size\_type](#input\_desired\_size\_type) | The unit of measurement for the value specified for `desired_size`. Supported for attribute-based instance type selection only. Valid values: `units`, `vcpu`, `memory-mib` | `string` | `null` | no | | [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance will be EBS-optimized | `bool` | `null` | no | | [efa\_indices](#input\_efa\_indices) | The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true` | `list(number)` |
[
0
]
| no | -| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | -| [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | -| [enable\_efa\_only](#input\_enable\_efa\_only) | Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later | `bool` | `false` | no | +| [enable\_efa\_only](#input\_enable\_efa\_only) | Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later | `bool` | `true` | no | | [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | -| [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | +| [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `false` | no | | [enabled\_metrics](#input\_enabled\_metrics) | A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances` | `list(string)` | `[]` | no | -| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | +| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances |
object({
enabled = optional(bool)
})
| `null` | no | | [force\_delete](#input\_force\_delete) | Allows deleting the Auto Scaling Group without waiting for all instances in the pool to terminate. You can force an Auto Scaling Group to delete even if it's in the process of scaling a resource. Normally, Terraform drains all the instances before deleting the group. This bypasses that behavior and potentially leaves resources dangling | `bool` | `null` | no | -| [force\_delete\_warm\_pool](#input\_force\_delete\_warm\_pool) | Allows deleting the Auto Scaling Group without waiting for all instances in the warm pool to terminate | `bool` | `null` | no | | [health\_check\_grace\_period](#input\_health\_check\_grace\_period) | Time (in seconds) after instance comes into service before checking health | `number` | `null` | no | | [health\_check\_type](#input\_health\_check\_type) | `EC2` or `ELB`. Controls how health checking is done | `string` | `null` | no | -| [hibernation\_options](#input\_hibernation\_options) | The hibernation options for the instance | `map(string)` | `{}` | no | | [iam\_instance\_profile\_arn](#input\_iam\_instance\_profile\_arn) | Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false` | `string` | `null` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | ARN of the IAM role used by the instance profile. Required when `create_access_entry = true` and `create_iam_instance_profile = false` | `string` | `null` | no | | [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | -| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `"Self managed node group IAM role"` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | -| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed |
list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
}))
| `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | -| [ignore\_failed\_scaling\_activities](#input\_ignore\_failed\_scaling\_activities) | Whether to ignore failed Auto Scaling scaling activities while waiting for capacity. | `bool` | `null` | no | -| [initial\_lifecycle\_hooks](#input\_initial\_lifecycle\_hooks) | One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource | `list(map(string))` | `[]` | no | +| [ignore\_failed\_scaling\_activities](#input\_ignore\_failed\_scaling\_activities) | Whether to ignore failed Auto Scaling scaling activities while waiting for capacity | `bool` | `null` | no | +| [initial\_lifecycle\_hooks](#input\_initial\_lifecycle\_hooks) | One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource |
list(object({
default_result = optional(string)
heartbeat_timeout = optional(number)
lifecycle_transition = string
name = string
notification_metadata = optional(string)
notification_target_arn = optional(string)
role_arn = optional(string)
}))
| `null` | no | | [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | -| [instance\_maintenance\_policy](#input\_instance\_maintenance\_policy) | If this block is configured, add a instance maintenance policy to the specified Auto Scaling group | `any` | `{}` | no | -| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `{}` | no | -| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated | `any` |
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | -| [instance\_requirements](#input\_instance\_requirements) | The attribute requirements for the type of instance. If present then `instance_type` cannot be present | `any` | `{}` | no | -| [instance\_type](#input\_instance\_type) | The type of the instance to launch | `string` | `""` | no | +| [instance\_maintenance\_policy](#input\_instance\_maintenance\_policy) | If this block is configured, add a instance maintenance policy to the specified Auto Scaling group |
object({
max_healthy_percentage = number
min_healthy_percentage = number
})
| `null` | no | +| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance |
object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
})
| `null` | no | +| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated |
object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number, 33)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string, "Rolling")
triggers = optional(list(string))
})
|
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | +| [instance\_requirements](#input\_instance\_requirements) | The attribute requirements for the type of instance. If present then `instance_type` cannot be present |
object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = string
}))
})
| `null` | no | +| [instance\_type](#input\_instance\_type) | The type of the instance to launch | `string` | `"m6i.large"` | no | | [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | | [key\_name](#input\_key\_name) | The key name that should be used for the instance | `string` | `null` | no | +| [kubernetes\_version](#input\_kubernetes\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | | [launch\_template\_default\_version](#input\_launch\_template\_default\_version) | Default Version of the launch template | `string` | `null` | no | | [launch\_template\_description](#input\_launch\_template\_description) | Description of the launch template | `string` | `null` | no | | [launch\_template\_id](#input\_launch\_template\_id) | The ID of an existing launch template to use. Required when `create_launch_template` = `false` | `string` | `""` | no | @@ -161,42 +157,42 @@ module "self_managed_node_group" { | [launch\_template\_tags](#input\_launch\_template\_tags) | A map of additional tags to add to the tag\_specifications of launch template created | `map(string)` | `{}` | no | | [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | | [launch\_template\_version](#input\_launch\_template\_version) | Launch template version. Can be version number, `$Latest`, or `$Default` | `string` | `null` | no | -| [license\_specifications](#input\_license\_specifications) | A map of license specifications to associate with | `any` | `{}` | no | -| [maintenance\_options](#input\_maintenance\_options) | The maintenance options for the instance | `any` | `{}` | no | +| [license\_specifications](#input\_license\_specifications) | A list of license specifications to associate with |
list(object({
license_configuration_arn = string
}))
| `null` | no | +| [maintenance\_options](#input\_maintenance\_options) | The maintenance options for the instance |
object({
auto_recovery = optional(string)
})
| `null` | no | | [max\_instance\_lifetime](#input\_max\_instance\_lifetime) | The maximum amount of time, in seconds, that an instance can be in service, values must be either equal to 0 or between 604800 and 31536000 seconds | `number` | `null` | no | | [max\_size](#input\_max\_size) | The maximum size of the autoscaling group | `number` | `3` | no | -| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | +| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance |
object({
http_endpoint = optional(string, "enabled")
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number, 1)
http_tokens = optional(string, "required")
instance_metadata_tags = optional(string)
})
|
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 1,
"http_tokens": "required"
}
| no | | [metrics\_granularity](#input\_metrics\_granularity) | The granularity to associate with the metrics to collect. The only valid value is `1Minute` | `string` | `null` | no | -| [min\_elb\_capacity](#input\_min\_elb\_capacity) | Setting this causes Terraform to wait for this number of instances to show up healthy in the ELB only on creation. Updates will not wait on ELB instance number changes | `number` | `null` | no | -| [min\_size](#input\_min\_size) | The minimum size of the autoscaling group | `number` | `0` | no | -| [mixed\_instances\_policy](#input\_mixed\_instances\_policy) | Configuration block containing settings to define launch targets for Auto Scaling groups | `any` | `null` | no | +| [min\_size](#input\_min\_size) | The minimum size of the autoscaling group | `number` | `1` | no | +| [mixed\_instances\_policy](#input\_mixed\_instances\_policy) | Configuration block containing settings to define launch targets for Auto Scaling groups |
object({
instances_distribution = optional(object({
on_demand_allocation_strategy = optional(string)
on_demand_base_capacity = optional(number)
on_demand_percentage_above_base_capacity = optional(number)
spot_allocation_strategy = optional(string)
spot_instance_pools = optional(number)
spot_max_price = optional(string)
}))
launch_template = object({
override = optional(list(object({
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = optional(number)
}))
}))
instance_type = optional(string)
launch_template_specification = optional(object({
launch_template_id = optional(string)
launch_template_name = optional(string)
version = optional(string)
}))
weighted_capacity = optional(string)
})))
})
})
| `null` | no | | [name](#input\_name) | Name of the Self managed Node Group | `string` | `""` | no | -| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | -| [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | -| [placement\_group](#input\_placement\_group) | The name of the placement group into which you'll launch your instances, if any | `string` | `null` | no | -| [placement\_group\_az](#input\_placement\_group\_az) | Availability zone where placement group is created (ex. `eu-west-1c`) | `string` | `null` | no | -| [platform](#input\_platform) | [DEPRECATED - must use `ami_type` instead. Will be removed in `v21.0`] | `string` | `null` | no | -| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | -| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | -| [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet | `map(string)` | `{}` | no | -| [protect\_from\_scale\_in](#input\_protect\_from\_scale\_in) | Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events. | `bool` | `false` | no | +| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time |
list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
subnet_id = optional(string)
}))
| `[]` | no | +| [partition](#input\_partition) | The AWS partition - pass through value to reduce number of GET requests from data sources | `string` | `""` | no | +| [placement](#input\_placement) | The placement of the instance |
object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
})
| `null` | no | +| [placement\_group](#input\_placement\_group) | The name of the placement group into which you'll launch your instances | `string` | `null` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `null` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `null` | no | +| [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet |
object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
})
| `null` | no | +| [protect\_from\_scale\_in](#input\_protect\_from\_scale\_in) | Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events | `bool` | `false` | no | | [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | -| [schedules](#input\_schedules) | Map of autoscaling group schedule to create | `map(any)` | `{}` | no | -| [service\_linked\_role\_arn](#input\_service\_linked\_role\_arn) | The ARN of the service-linked role that the ASG will use to call other AWS services | `string` | `null` | no | +| [region](#input\_region) | Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration | `string` | `null` | no | +| [security\_group\_description](#input\_security\_group\_description) | Description of the security group created | `string` | `null` | no | +| [security\_group\_egress\_rules](#input\_security\_group\_egress\_rules) | Security group egress rules to add to the security group created |
map(object({
name = optional(string)

cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string, "tcp")
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool, false)
tags = optional(map(string), {})
to_port = optional(string)
}))
| `{}` | no | +| [security\_group\_ingress\_rules](#input\_security\_group\_ingress\_rules) | Security group ingress rules to add to the security group created |
map(object({
name = optional(string)

cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string, "tcp")
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool, false)
tags = optional(map(string), {})
to_port = optional(string)
}))
| `{}` | no | +| [security\_group\_name](#input\_security\_group\_name) | Name to use on security group created | `string` | `null` | no | +| [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the security group created | `map(string)` | `{}` | no | +| [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `bool` | `true` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones` | `list(string)` | `null` | no | | [suspended\_processes](#input\_suspended\_processes) | A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly | `list(string)` | `[]` | no | | [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | -| [target\_group\_arns](#input\_target\_group\_arns) | A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing | `list(string)` | `[]` | no | | [termination\_policies](#input\_termination\_policies) | A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default` | `list(string)` | `[]` | no | +| [timeouts](#input\_timeouts) | Timeout configurations for the autoscaling group |
object({
delete = optional(string)
})
| `null` | no | | [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update Default Version each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | | [use\_mixed\_instances\_policy](#input\_use\_mixed\_instances\_policy) | Determines whether to use a mixed instances policy in the autoscaling group or not | `bool` | `false` | no | | [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix | `bool` | `true` | no | -| [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | +| [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `null` | no | | [vpc\_security\_group\_ids](#input\_vpc\_security\_group\_ids) | A list of security group IDs to associate | `list(string)` | `[]` | no | -| [wait\_for\_capacity\_timeout](#input\_wait\_for\_capacity\_timeout) | A maximum duration that Terraform should wait for ASG instances to be healthy before timing out. (See also Waiting for Capacity below.) Setting this to '0' causes Terraform to skip all Capacity Waiting behavior. | `string` | `null` | no | -| [wait\_for\_elb\_capacity](#input\_wait\_for\_elb\_capacity) | Setting this will cause Terraform to wait for exactly this number of healthy instances in all attached load balancers on both create and update operations. Takes precedence over `min_elb_capacity` behavior. | `number` | `null` | no | -| [warm\_pool](#input\_warm\_pool) | If this block is configured, add a Warm Pool to the specified Auto Scaling group | `any` | `{}` | no | ## Outputs @@ -213,7 +209,6 @@ module "self_managed_node_group" { | [autoscaling\_group\_max\_size](#output\_autoscaling\_group\_max\_size) | The maximum size of the autoscaling group | | [autoscaling\_group\_min\_size](#output\_autoscaling\_group\_min\_size) | The minimum size of the autoscaling group | | [autoscaling\_group\_name](#output\_autoscaling\_group\_name) | The autoscaling group name | -| [autoscaling\_group\_schedule\_arns](#output\_autoscaling\_group\_schedule\_arns) | ARNs of autoscaling group schedules | | [autoscaling\_group\_vpc\_zone\_identifier](#output\_autoscaling\_group\_vpc\_zone\_identifier) | The VPC zone identifier | | [iam\_instance\_profile\_arn](#output\_iam\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | | [iam\_instance\_profile\_id](#output\_iam\_instance\_profile\_id) | Instance profile's ID | @@ -226,6 +221,7 @@ module "self_managed_node_group" { | [launch\_template\_id](#output\_launch\_template\_id) | The ID of the launch template | | [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | | [launch\_template\_name](#output\_launch\_template\_name) | The name of the launch template | -| [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | +| [security\_group\_arn](#output\_security\_group\_arn) | Amazon Resource Name (ARN) of the security group | +| [security\_group\_id](#output\_security\_group\_id) | ID of the security group | | [user\_data](#output\_user\_data) | Base64 encoded user data | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 24454ef74d..710d276602 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -1,5 +1,14 @@ -data "aws_partition" "current" {} -data "aws_caller_identity" "current" {} +data "aws_partition" "current" { + count = var.create && var.partition == "" ? 1 : 0 +} +data "aws_caller_identity" "current" { + count = var.create && var.account_id == "" ? 1 : 0 +} + +locals { + partition = try(data.aws_partition.current[0].partition, var.partition) + account_id = try(data.aws_caller_identity.current[0].account_id, var.account_id) +} ################################################################################ # AMI SSM Parameter @@ -7,58 +16,36 @@ data "aws_caller_identity" "current" {} locals { # Just to ensure templating doesn't fail when values are not provided - ssm_cluster_version = var.cluster_version != null ? var.cluster_version : "" - - # TODO - Temporary stopgap for backwards compatibility until v21.0 - ami_type_to_user_data_type = { - AL2_x86_64 = "linux" - AL2_x86_64_GPU = "linux" - AL2_ARM_64 = "linux" - BOTTLEROCKET_ARM_64 = "bottlerocket" - BOTTLEROCKET_x86_64 = "bottlerocket" - BOTTLEROCKET_ARM_64_FIPS = "bottlerocket" - BOTTLEROCKET_x86_64_FIPS = "bottlerocket" - BOTTLEROCKET_ARM_64_NVIDIA = "bottlerocket" - BOTTLEROCKET_x86_64_NVIDIA = "bottlerocket" - WINDOWS_CORE_2019_x86_64 = "windows" - WINDOWS_FULL_2019_x86_64 = "windows" - WINDOWS_CORE_2022_x86_64 = "windows" - WINDOWS_FULL_2022_x86_64 = "windows" - AL2023_x86_64_STANDARD = "al2023" - AL2023_ARM_64_STANDARD = "al2023" - AL2023_x86_64_NEURON = "al2023" - AL2023_x86_64_NVIDIA = "al2023" - AL2023_ARM_64_NVIDIA = "al2023" - } - - user_data_type = local.ami_type_to_user_data_type[var.ami_type] + ssm_kubernetes_version = var.kubernetes_version != null ? var.kubernetes_version : "" # Map the AMI type to the respective SSM param path ami_type_to_ssm_param = { - AL2_x86_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2/recommended/image_id" - AL2_x86_64_GPU = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-gpu/recommended/image_id" - AL2_ARM_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-arm64/recommended/image_id" - BOTTLEROCKET_ARM_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/arm64/latest/image_id" - BOTTLEROCKET_x86_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/x86_64/latest/image_id" - BOTTLEROCKET_ARM_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/arm64/latest/image_id" - BOTTLEROCKET_x86_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/x86_64/latest/image_id" - BOTTLEROCKET_ARM_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/arm64/latest/image_id" - BOTTLEROCKET_x86_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/x86_64/latest/image_id" - WINDOWS_CORE_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-${local.ssm_cluster_version}/image_id" - WINDOWS_FULL_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-${local.ssm_cluster_version}/image_id" - WINDOWS_CORE_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-EKS_Optimized-${local.ssm_cluster_version}/image_id" - WINDOWS_FULL_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-${local.ssm_cluster_version}/image_id" - AL2023_x86_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/standard/recommended/image_id" - AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/standard/recommended/image_id" - AL2023_x86_64_NEURON = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/neuron/recommended/image_id" - AL2023_x86_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/nvidia/recommended/image_id" - AL2023_ARM_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/nvidia/recommended/image_id" + AL2_x86_64 = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2/recommended/image_id" + AL2_x86_64_GPU = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2-gpu/recommended/image_id" + AL2_ARM_64 = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2-arm64/recommended/image_id" + BOTTLEROCKET_ARM_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}/arm64/latest/image_id" + BOTTLEROCKET_x86_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}/x86_64/latest/image_id" + BOTTLEROCKET_ARM_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}-fips/arm64/latest/image_id" + BOTTLEROCKET_x86_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}-fips/x86_64/latest/image_id" + BOTTLEROCKET_ARM_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}-nvidia/arm64/latest/image_id" + BOTTLEROCKET_x86_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_kubernetes_version}-nvidia/x86_64/latest/image_id" + WINDOWS_CORE_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-${local.ssm_kubernetes_version}/image_id" + WINDOWS_FULL_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-${local.ssm_kubernetes_version}/image_id" + WINDOWS_CORE_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-EKS_Optimized-${local.ssm_kubernetes_version}/image_id" + WINDOWS_FULL_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-${local.ssm_kubernetes_version}/image_id" + AL2023_x86_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2023/x86_64/standard/recommended/image_id" + AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2023/arm64/standard/recommended/image_id" + AL2023_x86_64_NEURON = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2023/x86_64/neuron/recommended/image_id" + AL2023_x86_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2023/x86_64/nvidia/recommended/image_id" + AL2023_ARM_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_kubernetes_version}/amazon-linux-2023/arm64/nvidia/recommended/image_id" } } data "aws_ssm_parameter" "ami" { count = var.create ? 1 : 0 + region = var.region + name = local.ami_type_to_ssm_param[var.ami_type] } @@ -70,7 +57,6 @@ module "user_data" { source = "../_user_data" create = var.create - platform = local.user_data_type ami_type = var.ami_type is_eks_managed_node_group = false @@ -98,6 +84,8 @@ module "user_data" { data "aws_ec2_instance_type" "this" { count = var.create && var.enable_efa_support ? 1 : 0 + region = var.region + instance_type = var.instance_type } @@ -128,70 +116,71 @@ locals { locals { launch_template_name = coalesce(var.launch_template_name, "${var.name}-node-group") security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) - - placement = local.enable_efa_support ? { group_name = aws_placement_group.this[0].name } : var.placement } resource "aws_launch_template" "this" { count = var.create && var.create_launch_template ? 1 : 0 + region = var.region + dynamic "block_device_mappings" { - for_each = var.block_device_mappings + for_each = var.block_device_mappings != null ? var.block_device_mappings : {} content { - device_name = try(block_device_mappings.value.device_name, null) + device_name = block_device_mappings.value.device_name dynamic "ebs" { - for_each = try([block_device_mappings.value.ebs], []) + for_each = block_device_mappings.value.ebs != null ? [block_device_mappings.value.ebs] : [] content { - delete_on_termination = try(ebs.value.delete_on_termination, null) - encrypted = try(ebs.value.encrypted, null) - iops = try(ebs.value.iops, null) - kms_key_id = try(ebs.value.kms_key_id, null) - snapshot_id = try(ebs.value.snapshot_id, null) - throughput = try(ebs.value.throughput, null) - volume_size = try(ebs.value.volume_size, null) - volume_type = try(ebs.value.volume_type, null) + delete_on_termination = ebs.value.delete_on_termination + encrypted = ebs.value.encrypted + iops = ebs.value.iops + kms_key_id = ebs.value.kms_key_id + snapshot_id = ebs.value.snapshot_id + throughput = ebs.value.throughput + volume_initialization_rate = ebs.value.volume_initialization_rate + volume_size = ebs.value.volume_size + volume_type = ebs.value.volume_type } } - no_device = try(block_device_mappings.value.no_device, null) - virtual_name = try(block_device_mappings.value.virtual_name, null) + no_device = block_device_mappings.value.no_device + virtual_name = block_device_mappings.value.virtual_name } } dynamic "capacity_reservation_specification" { - for_each = length(var.capacity_reservation_specification) > 0 ? [var.capacity_reservation_specification] : [] + for_each = var.capacity_reservation_specification != null ? [var.capacity_reservation_specification] : [] content { - capacity_reservation_preference = try(capacity_reservation_specification.value.capacity_reservation_preference, null) + capacity_reservation_preference = capacity_reservation_specification.value.capacity_reservation_preference dynamic "capacity_reservation_target" { - for_each = try([capacity_reservation_specification.value.capacity_reservation_target], []) - + for_each = capacity_reservation_specification.value.capacity_reservation_target != null ? [capacity_reservation_specification.value.capacity_reservation_target] : [] content { - capacity_reservation_id = try(capacity_reservation_target.value.capacity_reservation_id, null) - capacity_reservation_resource_group_arn = try(capacity_reservation_target.value.capacity_reservation_resource_group_arn, null) + capacity_reservation_id = capacity_reservation_target.value.capacity_reservation_id + capacity_reservation_resource_group_arn = capacity_reservation_target.value.capacity_reservation_resource_group_arn } } } } dynamic "cpu_options" { - for_each = length(var.cpu_options) > 0 ? [var.cpu_options] : [] + for_each = var.cpu_options != null ? [var.cpu_options] : [] content { - core_count = try(cpu_options.value.core_count, null) - threads_per_core = try(cpu_options.value.threads_per_core, null) + amd_sev_snp = cpu_options.value.amd_sev_snp + core_count = cpu_options.value.core_count + threads_per_core = cpu_options.value.threads_per_core } } dynamic "credit_specification" { - for_each = length(var.credit_specification) > 0 ? [var.credit_specification] : [] + for_each = var.credit_specification != null ? [var.credit_specification] : [] content { - cpu_credits = try(credit_specification.value.cpu_credits, null) + cpu_credits = credit_specification.value.cpu_credits } } @@ -200,38 +189,14 @@ resource "aws_launch_template" "this" { disable_api_termination = var.disable_api_termination ebs_optimized = var.ebs_optimized - dynamic "elastic_gpu_specifications" { - for_each = var.elastic_gpu_specifications - - content { - type = elastic_gpu_specifications.value.type - } - } - - dynamic "elastic_inference_accelerator" { - for_each = length(var.elastic_inference_accelerator) > 0 ? [var.elastic_inference_accelerator] : [] - - content { - type = elastic_inference_accelerator.value.type - } - } - dynamic "enclave_options" { - for_each = length(var.enclave_options) > 0 ? [var.enclave_options] : [] + for_each = var.enclave_options != null ? [var.enclave_options] : [] content { enabled = enclave_options.value.enabled } } - dynamic "hibernation_options" { - for_each = length(var.hibernation_options) > 0 ? [var.hibernation_options] : [] - - content { - configured = hibernation_options.value.configured - } - } - iam_instance_profile { arn = var.create_iam_instance_profile ? aws_iam_instance_profile.this[0].arn : var.iam_instance_profile_arn } @@ -240,125 +205,116 @@ resource "aws_launch_template" "this" { instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior dynamic "instance_market_options" { - for_each = length(var.instance_market_options) > 0 ? [var.instance_market_options] : [] + for_each = var.instance_market_options != null ? [var.instance_market_options] : [] content { - market_type = try(instance_market_options.value.market_type, null) + market_type = instance_market_options.value.market_type dynamic "spot_options" { - for_each = try([instance_market_options.value.spot_options], []) + for_each = instance_market_options.value.spot_options != null ? [instance_market_options.value.spot_options] : [] content { - block_duration_minutes = try(spot_options.value.block_duration_minutes, null) - instance_interruption_behavior = try(spot_options.value.instance_interruption_behavior, null) - max_price = try(spot_options.value.max_price, null) - spot_instance_type = try(spot_options.value.spot_instance_type, null) - valid_until = try(spot_options.value.valid_until, null) + block_duration_minutes = spot_options.value.block_duration_minutes + instance_interruption_behavior = spot_options.value.instance_interruption_behavior + max_price = spot_options.value.max_price + spot_instance_type = spot_options.value.spot_instance_type + valid_until = spot_options.value.valid_until } } } } dynamic "instance_requirements" { - for_each = length(var.instance_requirements) > 0 ? [var.instance_requirements] : [] + for_each = var.instance_requirements != null ? [var.instance_requirements] : [] content { - dynamic "accelerator_count" { - for_each = try([instance_requirements.value.accelerator_count], []) + for_each = instance_requirements.value.accelerator_count != null ? [instance_requirements.value.accelerator_count] : [] content { - max = try(accelerator_count.value.max, null) - min = try(accelerator_count.value.min, null) + max = accelerator_count.value.max + min = accelerator_count.value.min } } - accelerator_manufacturers = try(instance_requirements.value.accelerator_manufacturers, []) - accelerator_names = try(instance_requirements.value.accelerator_names, []) + accelerator_manufacturers = instance_requirements.value.accelerator_manufacturers + accelerator_names = instance_requirements.value.accelerator_names dynamic "accelerator_total_memory_mib" { - for_each = try([instance_requirements.value.accelerator_total_memory_mib], []) + for_each = instance_requirements.value.accelerator_total_memory_mib != null ? [instance_requirements.value.accelerator_total_memory_mib] : [] content { - max = try(accelerator_total_memory_mib.value.max, null) - min = try(accelerator_total_memory_mib.value.min, null) + max = accelerator_total_memory_mib.value.max + min = accelerator_total_memory_mib.value.min } } - accelerator_types = try(instance_requirements.value.accelerator_types, []) - allowed_instance_types = try(instance_requirements.value.allowed_instance_types, null) - bare_metal = try(instance_requirements.value.bare_metal, null) + accelerator_types = instance_requirements.value.accelerator_types + allowed_instance_types = instance_requirements.value.allowed_instance_types + bare_metal = instance_requirements.value.bare_metal dynamic "baseline_ebs_bandwidth_mbps" { - for_each = try([instance_requirements.value.baseline_ebs_bandwidth_mbps], []) + for_each = instance_requirements.value.baseline_ebs_bandwidth_mbps != null ? [instance_requirements.value.baseline_ebs_bandwidth_mbps] : [] content { - max = try(baseline_ebs_bandwidth_mbps.value.max, null) - min = try(baseline_ebs_bandwidth_mbps.value.min, null) + max = baseline_ebs_bandwidth_mbps.value.max + min = baseline_ebs_bandwidth_mbps.value.min } } - burstable_performance = try(instance_requirements.value.burstable_performance, null) - cpu_manufacturers = try(instance_requirements.value.cpu_manufacturers, []) - excluded_instance_types = try(instance_requirements.value.excluded_instance_types, null) - instance_generations = try(instance_requirements.value.instance_generations, []) - local_storage = try(instance_requirements.value.local_storage, null) - local_storage_types = try(instance_requirements.value.local_storage_types, []) + burstable_performance = instance_requirements.value.burstable_performance + cpu_manufacturers = instance_requirements.value.cpu_manufacturers + excluded_instance_types = instance_requirements.value.excluded_instance_types + instance_generations = instance_requirements.value.instance_generations + local_storage = instance_requirements.value.local_storage + local_storage_types = instance_requirements.value.local_storage_types + max_spot_price_as_percentage_of_optimal_on_demand_price = instance_requirements.value.max_spot_price_as_percentage_of_optimal_on_demand_price dynamic "memory_gib_per_vcpu" { - for_each = try([instance_requirements.value.memory_gib_per_vcpu], []) + for_each = instance_requirements.value.memory_gib_per_vcpu != null ? [instance_requirements.value.memory_gib_per_vcpu] : [] content { - max = try(memory_gib_per_vcpu.value.max, null) - min = try(memory_gib_per_vcpu.value.min, null) + max = memory_gib_per_vcpu.value.max + min = memory_gib_per_vcpu.value.min } } dynamic "memory_mib" { - for_each = [instance_requirements.value.memory_mib] + for_each = instance_requirements.value.memory_mib != null ? [instance_requirements.value.memory_mib] : [] content { - max = try(memory_mib.value.max, null) + max = memory_mib.value.max min = memory_mib.value.min } } - dynamic "network_bandwidth_gbps" { - for_each = try([instance_requirements.value.network_bandwidth_gbps], []) - - content { - max = try(network_bandwidth_gbps.value.max, null) - min = try(network_bandwidth_gbps.value.min, null) - } - } - dynamic "network_interface_count" { - for_each = try([instance_requirements.value.network_interface_count], []) + for_each = instance_requirements.value.network_interface_count != null ? [instance_requirements.value.network_interface_count] : [] content { - max = try(network_interface_count.value.max, null) - min = try(network_interface_count.value.min, null) + max = network_interface_count.value.max + min = network_interface_count.value.min } } - on_demand_max_price_percentage_over_lowest_price = try(instance_requirements.value.on_demand_max_price_percentage_over_lowest_price, null) - require_hibernate_support = try(instance_requirements.value.require_hibernate_support, null) - spot_max_price_percentage_over_lowest_price = try(instance_requirements.value.spot_max_price_percentage_over_lowest_price, null) + on_demand_max_price_percentage_over_lowest_price = instance_requirements.value.on_demand_max_price_percentage_over_lowest_price + require_hibernate_support = instance_requirements.value.require_hibernate_support + spot_max_price_percentage_over_lowest_price = instance_requirements.value.spot_max_price_percentage_over_lowest_price dynamic "total_local_storage_gb" { - for_each = try([instance_requirements.value.total_local_storage_gb], []) + for_each = instance_requirements.value.total_local_storage_gb != null ? [instance_requirements.value.total_local_storage_gb] : [] content { - max = try(total_local_storage_gb.value.max, null) - min = try(total_local_storage_gb.value.min, null) + max = total_local_storage_gb.value.max + min = total_local_storage_gb.value.min } } dynamic "vcpu_count" { - for_each = [instance_requirements.value.vcpu_count] + for_each = instance_requirements.value.vcpu_count != null ? [instance_requirements.value.vcpu_count] : [] content { - max = try(vcpu_count.value.max, null) + max = vcpu_count.value.max min = vcpu_count.value.min } } @@ -370,7 +326,7 @@ resource "aws_launch_template" "this" { key_name = var.key_name dynamic "license_specification" { - for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} + for_each = var.license_specifications != null ? var.license_specifications : [] content { license_configuration_arn = license_specification.value.license_configuration_arn @@ -378,22 +334,22 @@ resource "aws_launch_template" "this" { } dynamic "maintenance_options" { - for_each = length(var.maintenance_options) > 0 ? [var.maintenance_options] : [] + for_each = var.maintenance_options != null ? [var.maintenance_options] : [] content { - auto_recovery = try(maintenance_options.value.auto_recovery, null) + auto_recovery = maintenance_options.value.auto_recovery } } dynamic "metadata_options" { - for_each = length(var.metadata_options) > 0 ? [var.metadata_options] : [] + for_each = var.metadata_options != null ? [var.metadata_options] : [] content { - http_endpoint = try(metadata_options.value.http_endpoint, null) - http_protocol_ipv6 = try(metadata_options.value.http_protocol_ipv6, null) - http_put_response_hop_limit = try(metadata_options.value.http_put_response_hop_limit, null) - http_tokens = try(metadata_options.value.http_tokens, null) - instance_metadata_tags = try(metadata_options.value.instance_metadata_tags, null) + http_endpoint = metadata_options.value.http_endpoint + http_protocol_ipv6 = metadata_options.value.http_protocol_ipv6 + http_put_response_hop_limit = metadata_options.value.http_put_response_hop_limit + http_tokens = metadata_options.value.http_tokens + instance_metadata_tags = metadata_options.value.instance_metadata_tags } } @@ -409,42 +365,72 @@ resource "aws_launch_template" "this" { name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null dynamic "network_interfaces" { - for_each = local.network_interfaces + for_each = length(var.network_interfaces) > 0 ? var.network_interfaces : [] content { - associate_carrier_ip_address = try(network_interfaces.value.associate_carrier_ip_address, null) - associate_public_ip_address = try(network_interfaces.value.associate_public_ip_address, null) - delete_on_termination = try(network_interfaces.value.delete_on_termination, null) - description = try(network_interfaces.value.description, null) - device_index = try(network_interfaces.value.device_index, null) - interface_type = try(network_interfaces.value.interface_type, null) - ipv4_address_count = try(network_interfaces.value.ipv4_address_count, null) - ipv4_addresses = try(network_interfaces.value.ipv4_addresses, []) - ipv4_prefix_count = try(network_interfaces.value.ipv4_prefix_count, null) - ipv4_prefixes = try(network_interfaces.value.ipv4_prefixes, null) - ipv6_address_count = try(network_interfaces.value.ipv6_address_count, null) - ipv6_addresses = try(network_interfaces.value.ipv6_addresses, []) - ipv6_prefix_count = try(network_interfaces.value.ipv6_prefix_count, null) - ipv6_prefixes = try(network_interfaces.value.ipv6_prefixes, []) - network_card_index = try(network_interfaces.value.network_card_index, null) - network_interface_id = try(network_interfaces.value.network_interface_id, null) - primary_ipv6 = try(network_interfaces.value.primary_ipv6, null) - private_ip_address = try(network_interfaces.value.private_ip_address, null) + associate_carrier_ip_address = network_interfaces.value.associate_carrier_ip_address + associate_public_ip_address = network_interfaces.value.associate_public_ip_address + + dynamic "connection_tracking_specification" { + for_each = network_interfaces.value.connection_tracking_specification != null ? [network_interfaces.value.connection_tracking_specification] : [] + + content { + tcp_established_timeout = connection_tracking_specification.value.tcp_established_timeout + udp_stream_timeout = connection_tracking_specification.value.udp_stream_timeout + udp_timeout = connection_tracking_specification.value.udp_timeout + } + } + + delete_on_termination = network_interfaces.value.delete_on_termination + description = network_interfaces.value.description + device_index = network_interfaces.value.device_index + + dynamic "ena_srd_specification" { + for_each = network_interfaces.value.ena_srd_specification != null ? [network_interfaces.value.ena_srd_specification] : [] + + content { + ena_srd_enabled = ena_srd_specification.value.ena_srd_enabled + + dynamic "ena_srd_udp_specification" { + for_each = ena_srd_specification.value.ena_srd_udp_specification != null ? [ena_srd_specification.value.ena_srd_udp_specification] : [] + + content { + ena_srd_udp_enabled = ena_srd_udp_specification.value.ena_srd_udp_enabled + } + } + } + } + + interface_type = network_interfaces.value.interface_type + ipv4_address_count = network_interfaces.value.ipv4_address_count + ipv4_addresses = network_interfaces.value.ipv4_addresses + ipv4_prefix_count = network_interfaces.value.ipv4_prefix_count + ipv4_prefixes = network_interfaces.value.ipv4_prefixes + ipv6_address_count = network_interfaces.value.ipv6_address_count + ipv6_addresses = network_interfaces.value.ipv6_addresses + ipv6_prefix_count = network_interfaces.value.ipv6_prefix_count + ipv6_prefixes = network_interfaces.value.ipv6_prefixes + network_card_index = network_interfaces.value.network_card_index + network_interface_id = network_interfaces.value.network_interface_id + primary_ipv6 = network_interfaces.value.primary_ipv6 + private_ip_address = network_interfaces.value.private_ip_address # Ref: https://github.com/hashicorp/terraform-provider-aws/issues/4570 - security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) - subnet_id = try(network_interfaces.value.subnet_id, null) + security_groups = compact(concat(network_interfaces.value.security_groups, var.vpc_security_group_ids)) + # Set on EKS managed node group, will fail if set here + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics + # subnet_id = try(network_interfaces.value.subnet_id, null) } } dynamic "placement" { - for_each = length(local.placement) > 0 ? [local.placement] : [] + for_each = var.placement != null || local.create_placement_group ? [var.placement] : [] content { affinity = try(placement.value.affinity, null) - availability_zone = lookup(placement.value, "availability_zone", null) - group_name = lookup(placement.value, "group_name", null) - host_id = lookup(placement.value, "host_id", null) - host_resource_group_arn = lookup(placement.value, "host_resource_group_arn", null) + availability_zone = try(placement.value.availability_zone, null) + group_name = try(aws_placement_group.this[0].name, placement.value.group_name) + host_id = try(placement.value.host_id, null) + host_resource_group_arn = try(placement.value.host_resource_group_arn, null) partition_number = try(placement.value.partition_number, null) spread_domain = try(placement.value.spread_domain, null) tenancy = try(placement.value.tenancy, null) @@ -452,12 +438,12 @@ resource "aws_launch_template" "this" { } dynamic "private_dns_name_options" { - for_each = length(var.private_dns_name_options) > 0 ? [var.private_dns_name_options] : [] + for_each = var.private_dns_name_options != null ? [var.private_dns_name_options] : [] content { - enable_resource_name_dns_aaaa_record = try(private_dns_name_options.value.enable_resource_name_dns_aaaa_record, null) - enable_resource_name_dns_a_record = try(private_dns_name_options.value.enable_resource_name_dns_a_record, null) - hostname_type = try(private_dns_name_options.value.hostname_type, null) + enable_resource_name_dns_aaaa_record = private_dns_name_options.value.enable_resource_name_dns_aaaa_record + enable_resource_name_dns_a_record = private_dns_name_options.value.enable_resource_name_dns_a_record + hostname_type = private_dns_name_options.value.hostname_type } } @@ -503,35 +489,35 @@ locals { resource "aws_autoscaling_group" "this" { count = var.create && var.create_autoscaling_group ? 1 : 0 + region = var.region + availability_zones = var.availability_zones capacity_rebalance = var.capacity_rebalance context = var.context - default_cooldown = var.default_cooldown default_instance_warmup = var.default_instance_warmup desired_capacity = var.desired_size desired_capacity_type = var.desired_size_type enabled_metrics = var.enabled_metrics force_delete = var.force_delete - force_delete_warm_pool = var.force_delete_warm_pool health_check_grace_period = var.health_check_grace_period health_check_type = var.health_check_type dynamic "initial_lifecycle_hook" { - for_each = var.initial_lifecycle_hooks + for_each = var.initial_lifecycle_hooks != null ? var.initial_lifecycle_hooks : [] content { - default_result = try(initial_lifecycle_hook.value.default_result, null) - heartbeat_timeout = try(initial_lifecycle_hook.value.heartbeat_timeout, null) + default_result = initial_lifecycle_hook.value.default_result + heartbeat_timeout = initial_lifecycle_hook.value.heartbeat_timeout lifecycle_transition = initial_lifecycle_hook.value.lifecycle_transition name = initial_lifecycle_hook.value.name - notification_metadata = try(initial_lifecycle_hook.value.notification_metadata, null) - notification_target_arn = try(initial_lifecycle_hook.value.notification_target_arn, null) - role_arn = try(initial_lifecycle_hook.value.role_arn, null) + notification_metadata = initial_lifecycle_hook.value.notification_metadata + notification_target_arn = initial_lifecycle_hook.value.notification_target_arn + role_arn = initial_lifecycle_hook.value.role_arn } } dynamic "instance_maintenance_policy" { - for_each = length(var.instance_maintenance_policy) > 0 ? [var.instance_maintenance_policy] : [] + for_each = var.instance_maintenance_policy != null ? [var.instance_maintenance_policy] : [] content { min_healthy_percentage = instance_maintenance_policy.value.min_healthy_percentage @@ -540,26 +526,35 @@ resource "aws_autoscaling_group" "this" { } dynamic "instance_refresh" { - for_each = length(var.instance_refresh) > 0 ? [var.instance_refresh] : [] + for_each = var.instance_refresh != null ? [var.instance_refresh] : [] content { dynamic "preferences" { - for_each = try([instance_refresh.value.preferences], []) + for_each = instance_refresh.value.preferences != null ? [instance_refresh.value.preferences] : [] content { - checkpoint_delay = try(preferences.value.checkpoint_delay, null) - checkpoint_percentages = try(preferences.value.checkpoint_percentages, null) - instance_warmup = try(preferences.value.instance_warmup, null) - max_healthy_percentage = try(preferences.value.max_healthy_percentage, null) - min_healthy_percentage = try(preferences.value.min_healthy_percentage, null) - scale_in_protected_instances = try(preferences.value.scale_in_protected_instances, null) - skip_matching = try(preferences.value.skip_matching, null) - standby_instances = try(preferences.value.standby_instances, null) + dynamic "alarm_specification" { + for_each = preferences.value.alarm_specification != null ? [preferences.value.alarm_specification] : [] + + content { + alarms = alarm_specification.value.alarms + } + } + + auto_rollback = preferences.value.auto_rollback + checkpoint_delay = preferences.value.checkpoint_delay + checkpoint_percentages = preferences.value.checkpoint_percentages + instance_warmup = preferences.value.instance_warmup + max_healthy_percentage = preferences.value.max_healthy_percentage + min_healthy_percentage = preferences.value.min_healthy_percentage + scale_in_protected_instances = preferences.value.scale_in_protected_instances + skip_matching = preferences.value.skip_matching + standby_instances = preferences.value.standby_instances } } strategy = instance_refresh.value.strategy - triggers = try(instance_refresh.value.triggers, null) + triggers = instance_refresh.value.triggers } } @@ -575,7 +570,6 @@ resource "aws_autoscaling_group" "this" { max_instance_lifetime = var.max_instance_lifetime max_size = var.max_size metrics_granularity = var.metrics_granularity - min_elb_capacity = var.min_elb_capacity min_size = var.min_size ignore_failed_scaling_activities = var.ignore_failed_scaling_activities @@ -585,149 +579,162 @@ resource "aws_autoscaling_group" "this" { content { dynamic "instances_distribution" { - for_each = try([mixed_instances_policy.value.instances_distribution], []) + for_each = mixed_instances_policy.value.instances_distribution != null ? [mixed_instances_policy.value.instances_distribution] : [] content { - on_demand_allocation_strategy = try(instances_distribution.value.on_demand_allocation_strategy, null) - on_demand_base_capacity = try(instances_distribution.value.on_demand_base_capacity, null) - on_demand_percentage_above_base_capacity = try(instances_distribution.value.on_demand_percentage_above_base_capacity, null) - spot_allocation_strategy = try(instances_distribution.value.spot_allocation_strategy, null) - spot_instance_pools = try(instances_distribution.value.spot_instance_pools, null) - spot_max_price = try(instances_distribution.value.spot_max_price, null) + on_demand_allocation_strategy = instances_distribution.value.on_demand_allocation_strategy + on_demand_base_capacity = instances_distribution.value.on_demand_base_capacity + on_demand_percentage_above_base_capacity = instances_distribution.value.on_demand_percentage_above_base_capacity + spot_allocation_strategy = instances_distribution.value.spot_allocation_strategy + spot_instance_pools = instances_distribution.value.spot_instance_pools + spot_max_price = instances_distribution.value.spot_max_price } } - launch_template { - launch_template_specification { - launch_template_id = local.launch_template_id - version = local.launch_template_version - } + dynamic "launch_template" { + for_each = [mixed_instances_policy.value.launch_template] - dynamic "override" { - for_each = try(mixed_instances_policy.value.override, []) + content { + launch_template_specification { + launch_template_id = local.launch_template_id + version = local.launch_template_version + } - content { - dynamic "instance_requirements" { - for_each = try([override.value.instance_requirements], []) + dynamic "override" { + for_each = launch_template.value.override != null ? launch_template.value.override : [] - content { + content { + dynamic "instance_requirements" { + for_each = override.value.instance_requirements != null ? [override.value.instance_requirements] : [] - dynamic "accelerator_count" { - for_each = try([instance_requirements.value.accelerator_count], []) + content { + dynamic "accelerator_count" { + for_each = instance_requirements.value.accelerator_count != null ? [instance_requirements.value.accelerator_count] : [] - content { - max = try(accelerator_count.value.max, null) - min = try(accelerator_count.value.min, null) + content { + max = accelerator_count.value.max + min = accelerator_count.value.min + } } - } - accelerator_manufacturers = try(instance_requirements.value.accelerator_manufacturers, []) - accelerator_names = try(instance_requirements.value.accelerator_names, []) + accelerator_manufacturers = instance_requirements.value.accelerator_manufacturers + accelerator_names = instance_requirements.value.accelerator_names - dynamic "accelerator_total_memory_mib" { - for_each = try([instance_requirements.value.accelerator_total_memory_mib], []) + dynamic "accelerator_total_memory_mib" { + for_each = instance_requirements.value.accelerator_total_memory_mib != null ? [instance_requirements.value.accelerator_total_memory_mib] : [] - content { - max = try(accelerator_total_memory_mib.value.max, null) - min = try(accelerator_total_memory_mib.value.min, null) + content { + max = accelerator_total_memory_mib.value.max + min = accelerator_total_memory_mib.value.min + } } - } - accelerator_types = try(instance_requirements.value.accelerator_types, []) - allowed_instance_types = try(instance_requirements.value.allowed_instance_types, null) - bare_metal = try(instance_requirements.value.bare_metal, null) + accelerator_types = instance_requirements.value.accelerator_types + allowed_instance_types = instance_requirements.value.allowed_instance_types + bare_metal = instance_requirements.value.bare_metal - dynamic "baseline_ebs_bandwidth_mbps" { - for_each = try([instance_requirements.value.baseline_ebs_bandwidth_mbps], []) + dynamic "baseline_ebs_bandwidth_mbps" { + for_each = instance_requirements.value.baseline_ebs_bandwidth_mbps != null ? [instance_requirements.value.baseline_ebs_bandwidth_mbps] : [] - content { - max = try(baseline_ebs_bandwidth_mbps.value.max, null) - min = try(baseline_ebs_bandwidth_mbps.value.min, null) + content { + max = baseline_ebs_bandwidth_mbps.value.max + min = baseline_ebs_bandwidth_mbps.value.min + } } - } - burstable_performance = try(instance_requirements.value.burstable_performance, null) - cpu_manufacturers = try(instance_requirements.value.cpu_manufacturers, []) - excluded_instance_types = try(instance_requirements.value.excluded_instance_types, []) - instance_generations = try(instance_requirements.value.instance_generations, []) - local_storage = try(instance_requirements.value.local_storage, null) - local_storage_types = try(instance_requirements.value.local_storage_types, []) + burstable_performance = instance_requirements.value.burstable_performance + cpu_manufacturers = instance_requirements.value.cpu_manufacturers + excluded_instance_types = instance_requirements.value.excluded_instance_types + instance_generations = instance_requirements.value.instance_generations + local_storage = instance_requirements.value.local_storage + local_storage_types = instance_requirements.value.local_storage_types + max_spot_price_as_percentage_of_optimal_on_demand_price = instance_requirements.value.max_spot_price_as_percentage_of_optimal_on_demand_price + + dynamic "memory_gib_per_vcpu" { + for_each = instance_requirements.value.memory_gib_per_vcpu != null ? [instance_requirements.value.memory_gib_per_vcpu] : [] + + content { + max = memory_gib_per_vcpu.value.max + min = memory_gib_per_vcpu.value.min + } + } - dynamic "memory_gib_per_vcpu" { - for_each = try([instance_requirements.value.memory_gib_per_vcpu], []) + dynamic "memory_mib" { + for_each = instance_requirements.value.memory_mib != null ? [instance_requirements.value.memory_mib] : [] - content { - max = try(memory_gib_per_vcpu.value.max, null) - min = try(memory_gib_per_vcpu.value.min, null) + content { + max = memory_mib.value.max + min = memory_mib.value.min + } } - } - dynamic "memory_mib" { - for_each = [instance_requirements.value.memory_mib] + dynamic "network_bandwidth_gbps" { + for_each = instance_requirements.value.network_bandwidth_gbps != null ? [instance_requirements.value.network_bandwidth_gbps] : [] - content { - max = try(memory_mib.value.max, null) - min = memory_mib.value.min + content { + max = network_bandwidth_gbps.value.max + min = network_bandwidth_gbps.value.min + } } - } - dynamic "network_interface_count" { - for_each = try([instance_requirements.value.network_interface_count], []) + dynamic "network_interface_count" { + for_each = instance_requirements.value.network_interface_count != null ? [instance_requirements.value.network_interface_count] : [] - content { - max = try(network_interface_count.value.max, null) - min = try(network_interface_count.value.min, null) + content { + max = network_interface_count.value.max + min = network_interface_count.value.min + } } - } - on_demand_max_price_percentage_over_lowest_price = try(instance_requirements.value.on_demand_max_price_percentage_over_lowest_price, null) - require_hibernate_support = try(instance_requirements.value.require_hibernate_support, null) - spot_max_price_percentage_over_lowest_price = try(instance_requirements.value.spot_max_price_percentage_over_lowest_price, null) + on_demand_max_price_percentage_over_lowest_price = instance_requirements.value.on_demand_max_price_percentage_over_lowest_price + require_hibernate_support = instance_requirements.value.require_hibernate_support + spot_max_price_percentage_over_lowest_price = instance_requirements.value.spot_max_price_percentage_over_lowest_price - dynamic "total_local_storage_gb" { - for_each = try([instance_requirements.value.total_local_storage_gb], []) + dynamic "total_local_storage_gb" { + for_each = instance_requirements.value.total_local_storage_gb != null ? [instance_requirements.value.total_local_storage_gb] : [] - content { - max = try(total_local_storage_gb.value.max, null) - min = try(total_local_storage_gb.value.min, null) + content { + max = total_local_storage_gb.value.max + min = total_local_storage_gb.value.min + } } - } - dynamic "vcpu_count" { - for_each = [instance_requirements.value.vcpu_count] + dynamic "vcpu_count" { + for_each = instance_requirements.value.vcpu_count != null ? [instance_requirements.value.vcpu_count] : [] - content { - max = try(vcpu_count.value.max, null) - min = vcpu_count.value.min + content { + max = vcpu_count.value.max + min = vcpu_count.value.min + } } } } - } - instance_type = try(override.value.instance_type, null) + instance_type = override.value.instance_type - dynamic "launch_template_specification" { - for_each = try([override.value.launch_template_specification], []) + dynamic "launch_template_specification" { + for_each = override.value.launch_template_specification != null ? [override.value.launch_template_specification] : [] - content { - launch_template_id = try(launch_template_specification.value.launch_template_id, null) - version = try(launch_template_specification.value.version, null) + content { + launch_template_id = launch_template_specification.value.launch_template_id + launch_template_name = launch_template_specification.value.launch_template_name + version = launch_template_specification.value.version + } } - } - weighted_capacity = try(override.value.weighted_capacity, null) + weighted_capacity = override.value.weighted_capacity + } } } } } } - name = var.use_name_prefix ? null : var.name - name_prefix = var.use_name_prefix ? "${var.name}-" : null - placement_group = var.placement_group - protect_from_scale_in = var.protect_from_scale_in - service_linked_role_arn = var.service_linked_role_arn - suspended_processes = var.suspended_processes + name = var.use_name_prefix ? null : var.name + name_prefix = var.use_name_prefix ? "${var.name}-" : null + placement_group = var.placement_group + protect_from_scale_in = var.protect_from_scale_in + suspended_processes = var.suspended_processes dynamic "tag" { for_each = merge( @@ -756,34 +763,17 @@ resource "aws_autoscaling_group" "this" { } } - target_group_arns = var.target_group_arns - termination_policies = var.termination_policies - vpc_zone_identifier = local.enable_efa_support ? data.aws_subnets.placement_group[0].ids : var.subnet_ids - wait_for_capacity_timeout = var.wait_for_capacity_timeout - wait_for_elb_capacity = var.wait_for_elb_capacity + termination_policies = var.termination_policies + vpc_zone_identifier = var.subnet_ids - dynamic "warm_pool" { - for_each = length(var.warm_pool) > 0 ? [var.warm_pool] : [] + dynamic "timeouts" { + for_each = var.timeouts != null ? [var.timeouts] : [] content { - dynamic "instance_reuse_policy" { - for_each = try([warm_pool.value.instance_reuse_policy], []) - - content { - reuse_on_scale_in = try(instance_reuse_policy.value.reuse_on_scale_in, null) - } - } - - max_group_prepared_capacity = try(warm_pool.value.max_group_prepared_capacity, null) - min_size = try(warm_pool.value.min_size, null) - pool_state = try(warm_pool.value.pool_state, null) + delete = var.timeouts.delete } } - timeouts { - delete = var.delete_timeout - } - lifecycle { create_before_destroy = true ignore_changes = [ @@ -800,13 +790,13 @@ locals { create_iam_instance_profile = var.create && var.create_iam_instance_profile iam_role_name = coalesce(var.iam_role_name, "${var.name}-node-group") - iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" ipv4_cni_policy = { for k, v in { AmazonEKS_CNI_Policy = "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } ipv6_cni_policy = { for k, v in { - AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + AmazonEKS_CNI_IPv6_Policy = "arn:${local.partition}:iam::${local.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } } @@ -882,25 +872,25 @@ resource "aws_iam_instance_profile" "this" { ################################################################################ locals { - create_iam_role_policy = local.create_iam_instance_profile && var.create_iam_role_policy && length(var.iam_role_policy_statements) > 0 + create_iam_role_policy = local.create_iam_instance_profile && var.create_iam_role_policy && var.iam_role_policy_statements != null } data "aws_iam_policy_document" "role" { count = local.create_iam_role_policy ? 1 : 0 dynamic "statement" { - for_each = var.iam_role_policy_statements + for_each = var.iam_role_policy_statements != null ? var.iam_role_policy_statements : [] content { - sid = try(statement.value.sid, null) - actions = try(statement.value.actions, null) - not_actions = try(statement.value.not_actions, null) - effect = try(statement.value.effect, null) - resources = try(statement.value.resources, null) - not_resources = try(statement.value.not_resources, null) + sid = statement.value.sid + actions = statement.value.actions + not_actions = statement.value.not_actions + effect = statement.value.effect + resources = statement.value.resources + not_resources = statement.value.not_resources dynamic "principals" { - for_each = try(statement.value.principals, []) + for_each = statement.value.principals != null ? statement.value.principals : [] content { type = principals.value.type @@ -909,7 +899,7 @@ data "aws_iam_policy_document" "role" { } dynamic "not_principals" { - for_each = try(statement.value.not_principals, []) + for_each = statement.value.not_principals != null ? statement.value.not_principals : [] content { type = not_principals.value.type @@ -918,7 +908,7 @@ data "aws_iam_policy_document" "role" { } dynamic "condition" { - for_each = try(statement.value.conditions, []) + for_each = statement.value.condition != null ? statement.value.condition : [] content { test = condition.value.test @@ -950,6 +940,8 @@ locals { resource "aws_placement_group" "this" { count = local.create_placement_group ? 1 : 0 + region = var.region + name = "${var.cluster_name}-${var.name}" strategy = "cluster" @@ -957,91 +949,124 @@ resource "aws_placement_group" "this" { } ################################################################################ -# Instance AZ Lookup - -# Instances usually used in placement groups w/ EFA are only available in -# select availability zones. These data sources will cross reference the availability -# zones supported by the instance type with the subnets provided to ensure only -# AZs/subnets that are supported are used. +# Access Entry ################################################################################ -# Find the availability zones supported by the instance type -# TODO - remove at next breaking change -# Force users to be explicit about which AZ to use when using placement groups, -# with or without EFA support -data "aws_ec2_instance_type_offerings" "this" { - count = local.enable_efa_support ? 1 : 0 +resource "aws_eks_access_entry" "this" { + count = var.create && var.create_access_entry ? 1 : 0 + + region = var.region - filter { - name = "instance-type" - values = [var.instance_type] - } + cluster_name = var.cluster_name + principal_arn = var.create_iam_instance_profile ? aws_iam_role.this[0].arn : var.iam_role_arn + type = startswith(var.ami_type, "WINDOWS_") ? "EC2_WINDOWS" : "EC2_LINUX" - location_type = "availability-zone-id" + tags = var.tags } -# Reverse the lookup to find one of the subnets provided based on the availability -# availability zone ID of the queried instance type (supported) -data "aws_subnets" "placement_group" { - count = local.create_placement_group ? 1 : 0 +################################################################################ +# Security Group +################################################################################ - filter { - name = "subnet-id" - values = var.subnet_ids - } +locals { + create_security_group = var.create && var.create_security_group && length(merge(local.security_group_ingress_rules, local.security_group_egress_rules)) > 0 + security_group_name = coalesce(var.security_group_name, "${var.cluster_name}-${var.name}") - # The data source can lookup the first available AZ or you can specify an AZ (next filter) - dynamic "filter" { - for_each = local.create_placement_group && var.placement_group_az == null ? [1] : [] + security_group_ingress_rules = merge({ for k, v in + { + all_self_efa = { + description = "Node to node EFA" + protocol = "-1" + from_port = 0 + self = true + } + } : k => v if var.enable_efa_support + }, + var.security_group_ingress_rules + ) + security_group_egress_rules = merge({ for k, v in + { + all_self_efa = { + description = "Node to node EFA" + protocol = "-1" + to_port = 0 + self = true + } + } : k => v if var.enable_efa_support + }, + var.security_group_egress_rules + ) +} - content { - name = "availability-zone-id" - values = data.aws_ec2_instance_type_offerings.this[0].locations - } - } +data "aws_subnet" "this" { + count = local.create_security_group ? 1 : 0 - dynamic "filter" { - for_each = var.placement_group_az != null ? [var.placement_group_az] : [] + region = var.region - content { - name = "availability-zone" - values = [filter.value] - } - } + id = element(var.subnet_ids, 0) } -################################################################################ -# Access Entry -################################################################################ - -resource "aws_eks_access_entry" "this" { - count = var.create && var.create_access_entry ? 1 : 0 +resource "aws_security_group" "this" { + count = local.create_security_group ? 1 : 0 - cluster_name = var.cluster_name - principal_arn = var.create_iam_instance_profile ? aws_iam_role.this[0].arn : var.iam_role_arn - type = local.user_data_type == "windows" ? "EC2_WINDOWS" : "EC2_LINUX" + region = var.region - tags = var.tags -} + name = var.security_group_use_name_prefix ? null : local.security_group_name + name_prefix = var.security_group_use_name_prefix ? "${local.security_group_name}-" : null + description = var.security_group_description + vpc_id = data.aws_subnet.this[0].vpc_id -################################################################################ -# Autoscaling group schedule -################################################################################ - -resource "aws_autoscaling_schedule" "this" { - for_each = { for k, v in var.schedules : k => v if var.create && var.create_schedule } + tags = merge( + var.tags, + { "Name" = local.security_group_name }, + var.security_group_tags + ) - scheduled_action_name = each.key - autoscaling_group_name = aws_autoscaling_group.this[0].name + lifecycle { + create_before_destroy = true + } +} - min_size = try(each.value.min_size, null) - max_size = try(each.value.max_size, null) - desired_capacity = try(each.value.desired_size, null) - start_time = try(each.value.start_time, null) - end_time = try(each.value.end_time, null) - time_zone = try(each.value.time_zone, null) +resource "aws_vpc_security_group_ingress_rule" "this" { + for_each = { for k, v in local.security_group_ingress_rules : k => v if length(local.security_group_ingress_rules) > 0 && local.create_security_group } + + region = var.region + + cidr_ipv4 = each.value.cidr_ipv4 + cidr_ipv6 = each.value.cidr_ipv6 + description = each.value.description + from_port = each.value.from_port + ip_protocol = each.value.ip_protocol + prefix_list_id = each.value.prefix_list_id + referenced_security_group_id = each.value.self ? aws_security_group.this[0].id : each.value.referenced_security_group_id + security_group_id = aws_security_group.this[0].id + tags = merge( + var.tags, + var.security_group_tags, + { "Name" = coalesce(each.value.name, "${local.security_group_name}-${each.key}") }, + each.value.tags + ) + to_port = try(coalesce(each.value.to_port, each.value.from_port), null) +} - # [Minute] [Hour] [Day_of_Month] [Month_of_Year] [Day_of_Week] - # Cron examples: https://crontab.guru/examples.html - recurrence = try(each.value.recurrence, null) +resource "aws_vpc_security_group_egress_rule" "this" { + for_each = { for k, v in local.security_group_egress_rules : k => v if length(local.security_group_egress_rules) > 0 && local.create_security_group } + + region = var.region + + cidr_ipv4 = each.value.cidr_ipv4 + cidr_ipv6 = each.value.cidr_ipv6 + description = each.value.description + from_port = try(coalesce(each.value.from_port, each.value.to_port), null) + ip_protocol = each.value.ip_protocol + prefix_list_id = each.value.prefix_list_id + referenced_security_group_id = each.value.self ? aws_security_group.this[0].id : each.value.referenced_security_group_id + security_group_id = aws_security_group.this[0].id + tags = merge( + var.tags, + var.security_group_tags, + { "Name" = coalesce(each.value.name, "${local.security_group_name}-${each.key}") }, + each.value.tags + ) + to_port = each.value.to_port } diff --git a/modules/self-managed-node-group/outputs.tf b/modules/self-managed-node-group/outputs.tf index 9607810ac3..ad8710b890 100644 --- a/modules/self-managed-node-group/outputs.tf +++ b/modules/self-managed-node-group/outputs.tf @@ -128,24 +128,10 @@ output "access_entry_arn" { value = try(aws_eks_access_entry.this[0].access_entry_arn, null) } -################################################################################ -# Autoscaling Group Schedule -################################################################################ - -output "autoscaling_group_schedule_arns" { - description = "ARNs of autoscaling group schedules" - value = { for k, v in aws_autoscaling_schedule.this : k => v.arn } -} - ################################################################################ # Additional ################################################################################ -output "platform" { - description = "[DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" - value = module.user_data.platform -} - output "image_id" { description = "ID of the image" value = try(aws_launch_template.this[0].image_id, null) @@ -155,3 +141,17 @@ output "user_data" { description = "Base64 encoded user data" value = try(module.user_data.user_data, null) } + +################################################################################ +# Security Group +################################################################################ + +output "security_group_arn" { + description = "Amazon Resource Name (ARN) of the security group" + value = try(aws_security_group.this[0].arn, null) +} + +output "security_group_id" { + description = "ID of the security group" + value = try(aws_security_group.this[0].id, null) +} diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index 9076dab5af..a07d429f6b 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -2,6 +2,7 @@ variable "create" { description = "Determines whether to create self managed node group or not" type = bool default = true + nullable = false } variable "tags" { @@ -10,16 +11,22 @@ variable "tags" { default = {} } -# tflint-ignore: terraform_unused_declarations -variable "platform" { - description = "[DEPRECATED - must use `ami_type` instead. Will be removed in `v21.0`]" +variable "region" { + description = "Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration" type = string default = null +} - validation { - condition = var.platform == null - error_message = "`platform` is no longer valid due to the number of OS choices. Please provide an [`ami_type`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-amitype) instead." - } +variable "partition" { + description = "The AWS partition - pass through value to reduce number of GET requests from data sources" + type = string + default = "" +} + +variable "account_id" { + description = "The AWS account ID - pass through value to reduce number of GET requests from data sources" + type = string + default = "" } ################################################################################ @@ -35,55 +42,55 @@ variable "cluster_name" { variable "cluster_endpoint" { description = "Endpoint of associated EKS cluster" type = string - default = "" + default = null } variable "cluster_auth_base64" { description = "Base64 encoded CA of associated EKS cluster" type = string - default = "" + default = null } variable "cluster_service_cidr" { description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" type = string - default = "" + default = null } variable "cluster_ip_family" { description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" type = string - default = "ipv4" + default = null } variable "additional_cluster_dns_ips" { description = "Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*`" type = list(string) - default = [] + default = null } variable "pre_bootstrap_user_data" { description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string - default = "" + default = null } variable "post_bootstrap_user_data" { description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" type = string - default = "" + default = null } variable "bootstrap_extra_args" { description = "Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" type = string - default = "" + default = null } variable "user_data_template_path" { description = "Path to a local, custom user data template file to use when rendering user data" type = string - default = "" + default = null } variable "cloudinit_pre_nodeadm" { @@ -94,7 +101,7 @@ variable "cloudinit_pre_nodeadm" { filename = optional(string) merge_type = optional(string) })) - default = [] + default = null } variable "cloudinit_post_nodeadm" { @@ -105,7 +112,7 @@ variable "cloudinit_post_nodeadm" { filename = optional(string) merge_type = optional(string) })) - default = [] + default = null } ################################################################################ @@ -116,6 +123,7 @@ variable "create_launch_template" { description = "Determines whether to create launch template or not" type = bool default = true + nullable = false } variable "launch_template_id" { @@ -134,6 +142,7 @@ variable "launch_template_use_name_prefix" { description = "Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix" type = bool default = true + nullable = false } variable "launch_template_description" { @@ -152,6 +161,7 @@ variable "update_launch_template_default_version" { description = "Whether to update Default Version each update. Conflicts with `launch_template_default_version`" type = bool default = true + nullable = false } variable "disable_api_termination" { @@ -180,92 +190,163 @@ variable "ram_disk_id" { variable "block_device_mappings" { description = "Specify volumes to attach to the instance besides the volumes specified by the AMI" - type = any - default = {} + type = map(object({ + device_name = optional(string) + ebs = optional(object({ + delete_on_termination = optional(bool) + encrypted = optional(bool) + iops = optional(number) + kms_key_id = optional(string) + snapshot_id = optional(string) + throughput = optional(number) + volume_initialization_rate = optional(number) + volume_size = optional(number) + volume_type = optional(string) + })) + no_device = optional(string) + virtual_name = optional(string) + })) + default = null } variable "capacity_reservation_specification" { description = "Targeting for EC2 capacity reservations" - type = any - default = {} + type = object({ + capacity_reservation_preference = optional(string) + capacity_reservation_target = optional(object({ + capacity_reservation_id = optional(string) + capacity_reservation_resource_group_arn = optional(string) + })) + }) + default = null } variable "cpu_options" { description = "The CPU options for the instance" - type = map(string) - default = {} + type = object({ + amd_sev_snp = optional(string) + core_count = optional(number) + threads_per_core = optional(number) + }) + default = null } variable "credit_specification" { description = "Customize the credit specification of the instance" - type = map(string) - default = {} -} - -variable "elastic_gpu_specifications" { - description = "The elastic GPU to attach to the instance" - type = any - default = {} -} - -variable "elastic_inference_accelerator" { - description = "Configuration block containing an Elastic Inference Accelerator to attach to the instance" - type = map(string) - default = {} + type = object({ + cpu_credits = optional(string) + }) + default = null } variable "enclave_options" { description = "Enable Nitro Enclaves on launched instances" - type = map(string) - default = {} -} - -variable "hibernation_options" { - description = "The hibernation options for the instance" - type = map(string) - default = {} + type = object({ + enabled = optional(bool) + }) + default = null } variable "instance_market_options" { description = "The market (purchasing) option for the instance" - type = any - default = {} + type = object({ + market_type = optional(string) + spot_options = optional(object({ + block_duration_minutes = optional(number) + instance_interruption_behavior = optional(string) + max_price = optional(string) + spot_instance_type = optional(string) + valid_until = optional(string) + })) + }) + default = null } variable "maintenance_options" { description = "The maintenance options for the instance" - type = any - default = {} + type = object({ + auto_recovery = optional(string) + }) + default = null } variable "license_specifications" { - description = "A map of license specifications to associate with" - type = any - default = {} + description = "A list of license specifications to associate with" + type = list(object({ + license_configuration_arn = string + })) + default = null } variable "network_interfaces" { description = "Customize network interfaces to be attached at instance boot time" - type = list(any) - default = [] + type = list(object({ + associate_carrier_ip_address = optional(bool) + associate_public_ip_address = optional(bool) + connection_tracking_specification = optional(object({ + tcp_established_timeout = optional(number) + udp_stream_timeout = optional(number) + udp_timeout = optional(number) + })) + delete_on_termination = optional(bool) + description = optional(string) + device_index = optional(number) + ena_srd_specification = optional(object({ + ena_srd_enabled = optional(bool) + ena_srd_udp_specification = optional(object({ + ena_srd_udp_enabled = optional(bool) + })) + })) + interface_type = optional(string) + ipv4_address_count = optional(number) + ipv4_addresses = optional(list(string)) + ipv4_prefix_count = optional(number) + ipv4_prefixes = optional(list(string)) + ipv6_address_count = optional(number) + ipv6_addresses = optional(list(string)) + ipv6_prefix_count = optional(number) + ipv6_prefixes = optional(list(string)) + network_card_index = optional(number) + network_interface_id = optional(string) + primary_ipv6 = optional(bool) + private_ip_address = optional(string) + security_groups = optional(list(string), []) + subnet_id = optional(string) + })) + default = [] + nullable = false } variable "placement" { description = "The placement of the instance" - type = map(string) - default = {} + type = object({ + affinity = optional(string) + availability_zone = optional(string) + group_name = optional(string) + host_id = optional(string) + host_resource_group_arn = optional(string) + partition_number = optional(number) + spread_domain = optional(string) + tenancy = optional(string) + }) + default = null } variable "create_placement_group" { description = "Determines whether a placement group is created & used by the node group" type = bool default = false + nullable = false } variable "private_dns_name_options" { description = "The options for the instance hostname. The default values are inherited from the subnet" - type = map(string) - default = {} + type = object({ + enable_resource_name_dns_aaaa_record = optional(bool) + enable_resource_name_dns_a_record = optional(bool) + hostname_type = optional(string) + }) + default = null } variable "ebs_optimized" { @@ -283,10 +364,11 @@ variable "ami_id" { variable "ami_type" { description = "Type of Amazon Machine Image (AMI) associated with the node group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values" type = string - default = "AL2_x86_64" + default = "AL2023_x86_64_STANDARD" + nullable = false } -variable "cluster_version" { +variable "kubernetes_version" { description = "Kubernetes cluster version - used to lookup default AMI ID if one is not provided" type = string default = null @@ -294,14 +376,67 @@ variable "cluster_version" { variable "instance_requirements" { description = "The attribute requirements for the type of instance. If present then `instance_type` cannot be present" - type = any - default = {} + type = object({ + accelerator_count = optional(object({ + max = optional(number) + min = optional(number) + })) + accelerator_manufacturers = optional(list(string)) + accelerator_names = optional(list(string)) + accelerator_total_memory_mib = optional(object({ + max = optional(number) + min = optional(number) + })) + accelerator_types = optional(list(string)) + allowed_instance_types = optional(list(string)) + bare_metal = optional(string) + baseline_ebs_bandwidth_mbps = optional(object({ + max = optional(number) + min = optional(number) + })) + burstable_performance = optional(string) + cpu_manufacturers = optional(list(string)) + excluded_instance_types = optional(list(string)) + instance_generations = optional(list(string)) + local_storage = optional(string) + local_storage_types = optional(list(string)) + max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number) + memory_gib_per_vcpu = optional(object({ + max = optional(number) + min = optional(number) + })) + memory_mib = optional(object({ + max = optional(number) + min = optional(number) + })) + network_bandwidth_gbps = optional(object({ + max = optional(number) + min = optional(number) + })) + network_interface_count = optional(object({ + max = optional(number) + min = optional(number) + })) + on_demand_max_price_percentage_over_lowest_price = optional(number) + require_hibernate_support = optional(bool) + spot_max_price_percentage_over_lowest_price = optional(number) + total_local_storage_gb = optional(object({ + max = optional(number) + min = optional(number) + })) + vcpu_count = optional(object({ + max = optional(number) + min = string + })) + }) + default = null } variable "instance_type" { description = "The type of the instance to launch" type = string - default = "" + default = "m6i.large" + nullable = false } variable "key_name" { @@ -314,6 +449,7 @@ variable "vpc_security_group_ids" { description = "A list of security group IDs to associate" type = list(string) default = [] + nullable = false } variable "cluster_primary_security_group_id" { @@ -325,48 +461,60 @@ variable "cluster_primary_security_group_id" { variable "enable_monitoring" { description = "Enables/disables detailed monitoring" type = bool - default = true + default = false + nullable = false } variable "enable_efa_support" { description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" type = bool default = false + nullable = false } -# TODO - make this true by default at next breaking change (remove variable, only pass indices) variable "enable_efa_only" { description = "Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later" type = bool - default = false + default = true + nullable = false } variable "efa_indices" { description = "The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true`" type = list(number) default = [0] + nullable = false } variable "metadata_options" { description = "Customize the metadata options for the instance" - type = map(string) + type = object({ + http_endpoint = optional(string, "enabled") + http_protocol_ipv6 = optional(string) + http_put_response_hop_limit = optional(number, 1) + http_tokens = optional(string, "required") + instance_metadata_tags = optional(string) + }) default = { http_endpoint = "enabled" + http_put_response_hop_limit = 1 http_tokens = "required" - http_put_response_hop_limit = 2 } + nullable = false } variable "launch_template_tags" { description = "A map of additional tags to add to the tag_specifications of launch template created" type = map(string) default = {} + nullable = false } variable "tag_specifications" { description = "The tags to apply to the resources during launch" type = list(string) default = ["instance", "volume", "network-interface"] + nullable = false } ################################################################################ @@ -377,6 +525,7 @@ variable "create_autoscaling_group" { description = "Determines whether to create autoscaling group or not" type = bool default = true + nullable = false } variable "name" { @@ -389,6 +538,7 @@ variable "use_name_prefix" { description = "Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix" type = bool default = true + nullable = false } variable "launch_template_version" { @@ -403,12 +553,6 @@ variable "availability_zones" { default = null } -variable "placement_group_az" { - description = "Availability zone where placement group is created (ex. `eu-west-1c`)" - type = string - default = null -} - variable "subnet_ids" { description = "A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones`" type = list(string) @@ -418,19 +562,22 @@ variable "subnet_ids" { variable "min_size" { description = "The minimum size of the autoscaling group" type = number - default = 0 + default = 1 + nullable = false } variable "max_size" { description = "The maximum size of the autoscaling group" type = number default = 3 + nullable = false } variable "desired_size" { description = "The number of Amazon EC2 instances that should be running in the autoscaling group" type = number default = 1 + nullable = false } variable "desired_size_type" { @@ -440,7 +587,7 @@ variable "desired_size_type" { } variable "ignore_failed_scaling_activities" { - description = "Whether to ignore failed Auto Scaling scaling activities while waiting for capacity." + description = "Whether to ignore failed Auto Scaling scaling activities while waiting for capacity" type = bool default = null } @@ -457,30 +604,6 @@ variable "capacity_rebalance" { default = null } -variable "min_elb_capacity" { - description = "Setting this causes Terraform to wait for this number of instances to show up healthy in the ELB only on creation. Updates will not wait on ELB instance number changes" - type = number - default = null -} - -variable "wait_for_elb_capacity" { - description = "Setting this will cause Terraform to wait for exactly this number of healthy instances in all attached load balancers on both create and update operations. Takes precedence over `min_elb_capacity` behavior." - type = number - default = null -} - -variable "wait_for_capacity_timeout" { - description = "A maximum duration that Terraform should wait for ASG instances to be healthy before timing out. (See also Waiting for Capacity below.) Setting this to '0' causes Terraform to skip all Capacity Waiting behavior." - type = string - default = null -} - -variable "default_cooldown" { - description = "The amount of time, in seconds, after a scaling activity completes before another scaling activity can start" - type = number - default = null -} - variable "default_instance_warmup" { description = "Amount of time, in seconds, until a newly launched instance can contribute to the Amazon CloudWatch metrics. This delay lets an instance finish initializing before Amazon EC2 Auto Scaling aggregates instance metrics, resulting in more reliable usage data" type = number @@ -488,19 +611,14 @@ variable "default_instance_warmup" { } variable "protect_from_scale_in" { - description = "Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events." + description = "Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events" type = bool default = false -} - -variable "target_group_arns" { - description = "A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing" - type = list(string) - default = [] + nullable = false } variable "placement_group" { - description = "The name of the placement group into which you'll launch your instances, if any" + description = "The name of the placement group into which you'll launch your instances" type = string default = null } @@ -523,22 +641,18 @@ variable "force_delete" { default = null } -variable "force_delete_warm_pool" { - description = "Allows deleting the Auto Scaling Group without waiting for all instances in the warm pool to terminate" - type = bool - default = null -} - variable "termination_policies" { description = "A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default`" type = list(string) default = [] + nullable = false } variable "suspended_processes" { description = "A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly" type = list(string) default = [] + nullable = false } variable "max_instance_lifetime" { @@ -551,6 +665,7 @@ variable "enabled_metrics" { description = "A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances`" type = list(string) default = [] + nullable = false } variable "metrics_granularity" { @@ -559,63 +674,157 @@ variable "metrics_granularity" { default = null } -variable "service_linked_role_arn" { - description = "The ARN of the service-linked role that the ASG will use to call other AWS services" - type = string - default = null -} - variable "initial_lifecycle_hooks" { description = "One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource" - type = list(map(string)) - default = [] + type = list(object({ + default_result = optional(string) + heartbeat_timeout = optional(number) + lifecycle_transition = string + name = string + notification_metadata = optional(string) + notification_target_arn = optional(string) + role_arn = optional(string) + })) + default = null } variable "instance_maintenance_policy" { description = "If this block is configured, add a instance maintenance policy to the specified Auto Scaling group" - type = any - default = {} + type = object({ + max_healthy_percentage = number + min_healthy_percentage = number + }) + default = null } variable "instance_refresh" { description = "If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated" - type = any + type = object({ + preferences = optional(object({ + alarm_specification = optional(object({ + alarms = optional(list(string)) + })) + auto_rollback = optional(bool) + checkpoint_delay = optional(number) + checkpoint_percentages = optional(list(number)) + instance_warmup = optional(number) + max_healthy_percentage = optional(number) + min_healthy_percentage = optional(number, 33) + scale_in_protected_instances = optional(string) + skip_matching = optional(bool) + standby_instances = optional(string) + })) + strategy = optional(string, "Rolling") + triggers = optional(list(string)) + }) default = { strategy = "Rolling" preferences = { min_healthy_percentage = 66 } } + nullable = false } variable "use_mixed_instances_policy" { description = "Determines whether to use a mixed instances policy in the autoscaling group or not" type = bool default = false + nullable = false } variable "mixed_instances_policy" { description = "Configuration block containing settings to define launch targets for Auto Scaling groups" - type = any - default = null -} - -variable "warm_pool" { - description = "If this block is configured, add a Warm Pool to the specified Auto Scaling group" - type = any - default = {} -} - -variable "delete_timeout" { - description = "Delete timeout to wait for destroying autoscaling group" - type = string - default = null + type = object({ + instances_distribution = optional(object({ + on_demand_allocation_strategy = optional(string) + on_demand_base_capacity = optional(number) + on_demand_percentage_above_base_capacity = optional(number) + spot_allocation_strategy = optional(string) + spot_instance_pools = optional(number) + spot_max_price = optional(string) + })) + launch_template = object({ + override = optional(list(object({ + instance_requirements = optional(object({ + accelerator_count = optional(object({ + max = optional(number) + min = optional(number) + })) + accelerator_manufacturers = optional(list(string)) + accelerator_names = optional(list(string)) + accelerator_total_memory_mib = optional(object({ + max = optional(number) + min = optional(number) + })) + accelerator_types = optional(list(string)) + allowed_instance_types = optional(list(string)) + bare_metal = optional(string) + baseline_ebs_bandwidth_mbps = optional(object({ + max = optional(number) + min = optional(number) + })) + burstable_performance = optional(string) + cpu_manufacturers = optional(list(string)) + excluded_instance_types = optional(list(string)) + instance_generations = optional(list(string)) + local_storage = optional(string) + local_storage_types = optional(list(string)) + max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number) + memory_gib_per_vcpu = optional(object({ + max = optional(number) + min = optional(number) + })) + memory_mib = optional(object({ + max = optional(number) + min = optional(number) + })) + network_bandwidth_gbps = optional(object({ + max = optional(number) + min = optional(number) + })) + network_interface_count = optional(object({ + max = optional(number) + min = optional(number) + })) + on_demand_max_price_percentage_over_lowest_price = optional(number) + require_hibernate_support = optional(bool) + spot_max_price_percentage_over_lowest_price = optional(number) + total_local_storage_gb = optional(object({ + max = optional(number) + min = optional(number) + })) + vcpu_count = optional(object({ + max = optional(number) + min = optional(number) + })) + })) + instance_type = optional(string) + launch_template_specification = optional(object({ + launch_template_id = optional(string) + launch_template_name = optional(string) + version = optional(string) + })) + weighted_capacity = optional(string) + }))) + }) + }) + default = null +} + +variable "timeouts" { + description = "Timeout configurations for the autoscaling group" + type = object({ + delete = optional(string) + }) + default = null } variable "autoscaling_group_tags" { description = "A map of additional tags to add to the autoscaling group created. Tags are applied to the autoscaling group only and are NOT propagated to instances" type = map(string) default = {} + nullable = false } ################################################################################ @@ -626,6 +835,7 @@ variable "create_iam_instance_profile" { description = "Determines whether an IAM instance profile is created or to use an existing IAM instance profile" type = bool default = true + nullable = false } variable "iam_instance_profile_arn" { @@ -644,6 +854,7 @@ variable "iam_role_use_name_prefix" { description = "Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix" type = bool default = true + nullable = false } variable "iam_role_path" { @@ -655,7 +866,8 @@ variable "iam_role_path" { variable "iam_role_description" { description = "Description of the role" type = string - default = null + default = "Self managed node group IAM role" + nullable = false } variable "iam_role_permissions_boundary" { @@ -668,18 +880,21 @@ variable "iam_role_attach_cni_policy" { description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" type = bool default = true + nullable = false } variable "iam_role_additional_policies" { description = "Additional policies to be added to the IAM role" type = map(string) default = {} + nullable = false } variable "iam_role_tags" { description = "A map of additional tags to add to the IAM role created" type = map(string) default = {} + nullable = false } ################################################################################ @@ -690,12 +905,33 @@ variable "create_iam_role_policy" { description = "Determines whether an IAM role policy is created or not" type = bool default = true + nullable = false } variable "iam_role_policy_statements" { description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" - type = any - default = [] + type = list(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + })) + default = null } ################################################################################ @@ -706,6 +942,7 @@ variable "create_access_entry" { description = "Determines whether an access entry is created for the IAM role used by the node group" type = bool default = true + nullable = false } variable "iam_role_arn" { @@ -715,17 +952,78 @@ variable "iam_role_arn" { } ################################################################################ -# Autoscaling group schedule +# Security Group ################################################################################ -variable "create_schedule" { - description = "Determines whether to create autoscaling group schedule or not" +variable "create_security_group" { + description = "Determines if a security group is created" + type = bool + default = true + nullable = false +} + +variable "security_group_name" { + description = "Name to use on security group created" + type = string + default = null +} + +variable "security_group_use_name_prefix" { + description = "Determines whether the security group name (`security_group_name`) is used as a prefix" type = bool default = true + nullable = false +} + +variable "security_group_description" { + description = "Description of the security group created" + type = string + default = null +} + +variable "security_group_ingress_rules" { + description = "Security group ingress rules to add to the security group created" + type = map(object({ + name = optional(string) + + cidr_ipv4 = optional(string) + cidr_ipv6 = optional(string) + description = optional(string) + from_port = optional(string) + ip_protocol = optional(string, "tcp") + prefix_list_id = optional(string) + referenced_security_group_id = optional(string) + self = optional(bool, false) + tags = optional(map(string), {}) + to_port = optional(string) + })) + default = {} + nullable = false +} + +variable "security_group_egress_rules" { + description = "Security group egress rules to add to the security group created" + type = map(object({ + name = optional(string) + + cidr_ipv4 = optional(string) + cidr_ipv6 = optional(string) + description = optional(string) + from_port = optional(string) + ip_protocol = optional(string, "tcp") + prefix_list_id = optional(string) + referenced_security_group_id = optional(string) + self = optional(bool, false) + tags = optional(map(string), {}) + to_port = optional(string) + })) + default = {} + nullable = false } -variable "schedules" { - description = "Map of autoscaling group schedule to create" - type = map(any) +variable "security_group_tags" { + description = "A map of additional tags to add to the security group created" + type = map(string) default = {} + nullable = false } diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/node_groups.tf b/node_groups.tf index de7411f226..501da7e0ff 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -1,23 +1,4 @@ locals { - metadata_options = { - http_endpoint = "enabled" - http_tokens = "required" - http_put_response_hop_limit = 2 - } - - # EKS managed node group - default_update_config = { - max_unavailable_percentage = 33 - } - - # Self-managed node group - default_instance_refresh = { - strategy = "Rolling" - preferences = { - min_healthy_percentage = 66 - } - } - kubernetes_network_config = try(aws_eks_cluster.this[0].kubernetes_network_config[0], {}) } @@ -32,12 +13,12 @@ resource "time_sleep" "this" { create_duration = var.dataplane_wait_duration triggers = { - cluster_name = aws_eks_cluster.this[0].id - cluster_endpoint = aws_eks_cluster.this[0].endpoint - cluster_version = aws_eks_cluster.this[0].version - cluster_service_cidr = var.cluster_ip_family == "ipv6" ? try(local.kubernetes_network_config.service_ipv6_cidr, "") : try(local.kubernetes_network_config.service_ipv4_cidr, "") + name = aws_eks_cluster.this[0].id + endpoint = aws_eks_cluster.this[0].endpoint + kubernetes_version = aws_eks_cluster.this[0].version + service_cidr = var.ip_family == "ipv6" ? try(local.kubernetes_network_config.service_ipv6_cidr, "") : try(local.kubernetes_network_config.service_ipv4_cidr, "") - cluster_certificate_authority_data = aws_eks_cluster.this[0].certificate_authority[0].data + certificate_authority_data = aws_eks_cluster.this[0].certificate_authority[0].data } } @@ -87,7 +68,7 @@ resource "aws_iam_policy" "cni_ipv6_policy" { ################################################################################ locals { - node_sg_name = coalesce(var.node_security_group_name, "${var.cluster_name}-node") + node_sg_name = coalesce(var.node_security_group_name, "${var.name}-node") create_node_sg = var.create && var.create_node_security_group node_security_group_id = local.create_node_sg ? aws_security_group.node[0].id : var.node_security_group_id @@ -179,35 +160,16 @@ locals { to_port = 0 type = "egress" cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null + ipv6_cidr_blocks = var.ip_family == "ipv6" ? ["::/0"] : null } } : k => v if var.node_security_group_enable_recommended_rules } - - efa_security_group_rules = { for k, v in - { - ingress_all_self_efa = { - description = "Node to node EFA" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "ingress" - self = true - } - egress_all_self_efa = { - description = "Node to node EFA" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "egress" - self = true - } - } : k => v if var.enable_efa_support - } } resource "aws_security_group" "node" { count = local.create_node_sg ? 1 : 0 + region = var.region + name = var.node_security_group_use_name_prefix ? null : local.node_sg_name name_prefix = var.node_security_group_use_name_prefix ? "${local.node_sg_name}${var.prefix_separator}" : null description = var.node_security_group_description @@ -216,8 +178,8 @@ resource "aws_security_group" "node" { tags = merge( var.tags, { - "Name" = local.node_sg_name - "kubernetes.io/cluster/${var.cluster_name}" = "owned" + "Name" = local.node_sg_name + "kubernetes.io/cluster/${var.name}" = "owned" }, var.node_security_group_tags ) @@ -229,26 +191,24 @@ resource "aws_security_group" "node" { resource "aws_security_group_rule" "node" { for_each = { for k, v in merge( - local.efa_security_group_rules, local.node_security_group_rules, local.node_security_group_recommended_rules, var.node_security_group_additional_rules, ) : k => v if local.create_node_sg } - # Required - security_group_id = aws_security_group.node[0].id - protocol = each.value.protocol - from_port = each.value.from_port - to_port = each.value.to_port - type = each.value.type - - # Optional - description = lookup(each.value, "description", null) - cidr_blocks = lookup(each.value, "cidr_blocks", null) - ipv6_cidr_blocks = lookup(each.value, "ipv6_cidr_blocks", null) - prefix_list_ids = lookup(each.value, "prefix_list_ids", []) - self = lookup(each.value, "self", null) - source_security_group_id = try(each.value.source_cluster_security_group, false) ? local.cluster_security_group_id : lookup(each.value, "source_security_group_id", null) + region = var.region + + security_group_id = aws_security_group.node[0].id + protocol = each.value.protocol + from_port = each.value.from_port + to_port = each.value.to_port + type = each.value.type + description = try(each.value.description, null) + cidr_blocks = try(each.value.cidr_blocks, null) + ipv6_cidr_blocks = try(each.value.ipv6_cidr_blocks, null) + prefix_list_ids = try(each.value.prefix_list_ids, null) + self = try(each.value.self, null) + source_security_group_id = try(each.value.source_cluster_security_group, false) ? local.security_group_id : try(each.value.source_security_group_id, null) } ################################################################################ @@ -258,35 +218,42 @@ resource "aws_security_group_rule" "node" { module "fargate_profile" { source = "./modules/fargate-profile" - for_each = { for k, v in var.fargate_profiles : k => v if var.create && !local.create_outposts_local_cluster } + for_each = var.create && !local.create_outposts_local_cluster && var.fargate_profiles != null ? var.fargate_profiles : {} + + create = each.value.create + + region = var.region - create = try(each.value.create, true) + # Pass through values to reduce GET requests from data sources + partition = local.partition + account_id = local.account_id # Fargate Profile - cluster_name = time_sleep.this[0].triggers["cluster_name"] - cluster_ip_family = var.cluster_ip_family - name = try(each.value.name, each.key) - subnet_ids = try(each.value.subnet_ids, var.fargate_profile_defaults.subnet_ids, var.subnet_ids) - selectors = try(each.value.selectors, var.fargate_profile_defaults.selectors, []) - timeouts = try(each.value.timeouts, var.fargate_profile_defaults.timeouts, {}) + cluster_name = time_sleep.this[0].triggers["name"] + cluster_ip_family = var.ip_family + name = coalesce(each.value.name, each.key) + subnet_ids = coalesce(each.value.subnet_ids, var.subnet_ids) + selectors = each.value.selectors + timeouts = each.value.timeouts # IAM role - create_iam_role = try(each.value.create_iam_role, var.fargate_profile_defaults.create_iam_role, true) - iam_role_arn = try(each.value.iam_role_arn, var.fargate_profile_defaults.iam_role_arn, null) - iam_role_name = try(each.value.iam_role_name, var.fargate_profile_defaults.iam_role_name, null) - iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, var.fargate_profile_defaults.iam_role_use_name_prefix, true) - iam_role_path = try(each.value.iam_role_path, var.fargate_profile_defaults.iam_role_path, null) - iam_role_description = try(each.value.iam_role_description, var.fargate_profile_defaults.iam_role_description, "Fargate profile IAM role") - iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.fargate_profile_defaults.iam_role_permissions_boundary, null) - iam_role_tags = try(each.value.iam_role_tags, var.fargate_profile_defaults.iam_role_tags, {}) - iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.fargate_profile_defaults.iam_role_attach_cni_policy, true) - # To better understand why this `lookup()` logic is required, see: - # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 - iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.fargate_profile_defaults, "iam_role_additional_policies", {})) - create_iam_role_policy = try(each.value.create_iam_role_policy, var.fargate_profile_defaults.create_iam_role_policy, true) - iam_role_policy_statements = try(each.value.iam_role_policy_statements, var.fargate_profile_defaults.iam_role_policy_statements, []) - - tags = merge(var.tags, try(each.value.tags, var.fargate_profile_defaults.tags, {})) + create_iam_role = each.value.create_iam_role + iam_role_arn = each.value.iam_role_arn + iam_role_name = each.value.iam_role_name + iam_role_use_name_prefix = each.value.iam_role_use_name_prefix + iam_role_path = each.value.iam_role_path + iam_role_description = each.value.iam_role_description + iam_role_permissions_boundary = each.value.iam_role_permissions_boundary + iam_role_tags = each.value.iam_role_tags + iam_role_attach_cni_policy = each.value.iam_role_attach_cni_policy + iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", null) + create_iam_role_policy = each.value.create_iam_role_policy + iam_role_policy_statements = each.value.iam_role_policy_statements + + tags = merge( + var.tags, + each.value.tags, + ) } ################################################################################ @@ -296,120 +263,124 @@ module "fargate_profile" { module "eks_managed_node_group" { source = "./modules/eks-managed-node-group" - for_each = { for k, v in var.eks_managed_node_groups : k => v if var.create && !local.create_outposts_local_cluster } + for_each = var.create && !local.create_outposts_local_cluster && var.eks_managed_node_groups != null ? var.eks_managed_node_groups : {} + + create = each.value.create + + region = var.region - create = try(each.value.create, true) + # Pass through values to reduce GET requests from data sources + partition = local.partition + account_id = local.account_id - cluster_name = time_sleep.this[0].triggers["cluster_name"] - cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"]) + cluster_name = time_sleep.this[0].triggers["name"] + kubernetes_version = try(each.value.kubernetes_version, time_sleep.this[0].triggers["kubernetes_version"]) # EKS Managed Node Group - name = try(each.value.name, each.key) - use_name_prefix = try(each.value.use_name_prefix, var.eks_managed_node_group_defaults.use_name_prefix, true) - - subnet_ids = try(each.value.subnet_ids, var.eks_managed_node_group_defaults.subnet_ids, var.subnet_ids) - - min_size = try(each.value.min_size, var.eks_managed_node_group_defaults.min_size, 1) - max_size = try(each.value.max_size, var.eks_managed_node_group_defaults.max_size, 3) - desired_size = try(each.value.desired_size, var.eks_managed_node_group_defaults.desired_size, 1) - - ami_id = try(each.value.ami_id, var.eks_managed_node_group_defaults.ami_id, "") - ami_type = try(each.value.ami_type, var.eks_managed_node_group_defaults.ami_type, null) - ami_release_version = try(each.value.ami_release_version, var.eks_managed_node_group_defaults.ami_release_version, null) - use_latest_ami_release_version = try(each.value.use_latest_ami_release_version, var.eks_managed_node_group_defaults.use_latest_ami_release_version, false) - - capacity_type = try(each.value.capacity_type, var.eks_managed_node_group_defaults.capacity_type, null) - disk_size = try(each.value.disk_size, var.eks_managed_node_group_defaults.disk_size, null) - force_update_version = try(each.value.force_update_version, var.eks_managed_node_group_defaults.force_update_version, null) - instance_types = try(each.value.instance_types, var.eks_managed_node_group_defaults.instance_types, null) - labels = try(each.value.labels, var.eks_managed_node_group_defaults.labels, null) - node_repair_config = try(each.value.node_repair_config, var.eks_managed_node_group_defaults.node_repair_config, null) - remote_access = try(each.value.remote_access, var.eks_managed_node_group_defaults.remote_access, {}) - taints = try(each.value.taints, var.eks_managed_node_group_defaults.taints, {}) - update_config = try(each.value.update_config, var.eks_managed_node_group_defaults.update_config, local.default_update_config) - timeouts = try(each.value.timeouts, var.eks_managed_node_group_defaults.timeouts, {}) + name = coalesce(each.value.name, each.key) + use_name_prefix = each.value.use_name_prefix + + subnet_ids = coalesce(each.value.subnet_ids, var.subnet_ids) + + min_size = each.value.min_size + max_size = each.value.max_size + desired_size = each.value.desired_size + + ami_id = each.value.ami_id + ami_type = each.value.ami_type + ami_release_version = each.value.ami_release_version + use_latest_ami_release_version = each.value.use_latest_ami_release_version + + capacity_type = each.value.capacity_type + disk_size = each.value.disk_size + force_update_version = each.value.force_update_version + instance_types = each.value.instance_types + labels = each.value.labels + node_repair_config = each.value.node_repair_config + remote_access = each.value.remote_access + taints = each.value.taints + update_config = each.value.update_config + timeouts = each.value.timeouts # User data - platform = try(each.value.platform, var.eks_managed_node_group_defaults.platform, "linux") - cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") - cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") - cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr - cluster_ip_family = var.cluster_ip_family - cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") - enable_bootstrap_user_data = try(each.value.enable_bootstrap_user_data, var.eks_managed_node_group_defaults.enable_bootstrap_user_data, false) - pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.eks_managed_node_group_defaults.pre_bootstrap_user_data, "") - post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.eks_managed_node_group_defaults.post_bootstrap_user_data, "") - bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.eks_managed_node_group_defaults.bootstrap_extra_args, "") - user_data_template_path = try(each.value.user_data_template_path, var.eks_managed_node_group_defaults.user_data_template_path, "") - cloudinit_pre_nodeadm = try(each.value.cloudinit_pre_nodeadm, var.eks_managed_node_group_defaults.cloudinit_pre_nodeadm, []) - cloudinit_post_nodeadm = try(each.value.cloudinit_post_nodeadm, var.eks_managed_node_group_defaults.cloudinit_post_nodeadm, []) + cluster_endpoint = try(time_sleep.this[0].triggers["endpoint"], "") + cluster_auth_base64 = try(time_sleep.this[0].triggers["certificate_authority_data"], "") + cluster_ip_family = var.ip_family + cluster_service_cidr = try(time_sleep.this[0].triggers["service_cidr"], "") + enable_bootstrap_user_data = each.value.enable_bootstrap_user_data + pre_bootstrap_user_data = each.value.pre_bootstrap_user_data + post_bootstrap_user_data = each.value.post_bootstrap_user_data + bootstrap_extra_args = each.value.bootstrap_extra_args + user_data_template_path = each.value.user_data_template_path + cloudinit_pre_nodeadm = each.value.cloudinit_pre_nodeadm + cloudinit_post_nodeadm = each.value.cloudinit_post_nodeadm # Launch Template - create_launch_template = try(each.value.create_launch_template, var.eks_managed_node_group_defaults.create_launch_template, true) - use_custom_launch_template = try(each.value.use_custom_launch_template, var.eks_managed_node_group_defaults.use_custom_launch_template, true) - launch_template_id = try(each.value.launch_template_id, var.eks_managed_node_group_defaults.launch_template_id, "") - launch_template_name = try(each.value.launch_template_name, var.eks_managed_node_group_defaults.launch_template_name, each.key) - launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.eks_managed_node_group_defaults.launch_template_use_name_prefix, true) - launch_template_version = try(each.value.launch_template_version, var.eks_managed_node_group_defaults.launch_template_version, null) - launch_template_default_version = try(each.value.launch_template_default_version, var.eks_managed_node_group_defaults.launch_template_default_version, null) - update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.eks_managed_node_group_defaults.update_launch_template_default_version, true) - launch_template_description = try(each.value.launch_template_description, var.eks_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} EKS managed node group") - launch_template_tags = try(each.value.launch_template_tags, var.eks_managed_node_group_defaults.launch_template_tags, {}) - tag_specifications = try(each.value.tag_specifications, var.eks_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface"]) - - ebs_optimized = try(each.value.ebs_optimized, var.eks_managed_node_group_defaults.ebs_optimized, null) - key_name = try(each.value.key_name, var.eks_managed_node_group_defaults.key_name, null) - disable_api_termination = try(each.value.disable_api_termination, var.eks_managed_node_group_defaults.disable_api_termination, null) - kernel_id = try(each.value.kernel_id, var.eks_managed_node_group_defaults.kernel_id, null) - ram_disk_id = try(each.value.ram_disk_id, var.eks_managed_node_group_defaults.ram_disk_id, null) - - block_device_mappings = try(each.value.block_device_mappings, var.eks_managed_node_group_defaults.block_device_mappings, {}) - capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.eks_managed_node_group_defaults.capacity_reservation_specification, {}) - cpu_options = try(each.value.cpu_options, var.eks_managed_node_group_defaults.cpu_options, {}) - credit_specification = try(each.value.credit_specification, var.eks_managed_node_group_defaults.credit_specification, {}) - elastic_gpu_specifications = try(each.value.elastic_gpu_specifications, var.eks_managed_node_group_defaults.elastic_gpu_specifications, {}) - elastic_inference_accelerator = try(each.value.elastic_inference_accelerator, var.eks_managed_node_group_defaults.elastic_inference_accelerator, {}) - enclave_options = try(each.value.enclave_options, var.eks_managed_node_group_defaults.enclave_options, {}) - instance_market_options = try(each.value.instance_market_options, var.eks_managed_node_group_defaults.instance_market_options, {}) - license_specifications = try(each.value.license_specifications, var.eks_managed_node_group_defaults.license_specifications, {}) - metadata_options = try(each.value.metadata_options, var.eks_managed_node_group_defaults.metadata_options, local.metadata_options) - enable_monitoring = try(each.value.enable_monitoring, var.eks_managed_node_group_defaults.enable_monitoring, true) - enable_efa_support = try(each.value.enable_efa_support, var.eks_managed_node_group_defaults.enable_efa_support, false) - enable_efa_only = try(each.value.enable_efa_only, var.eks_managed_node_group_defaults.enable_efa_only, false) - efa_indices = try(each.value.efa_indices, var.eks_managed_node_group_defaults.efa_indices, [0]) - create_placement_group = try(each.value.create_placement_group, var.eks_managed_node_group_defaults.create_placement_group, false) - placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, {}) - placement_group_az = try(each.value.placement_group_az, var.eks_managed_node_group_defaults.placement_group_az, null) - placement_group_strategy = try(each.value.placement_group_strategy, var.eks_managed_node_group_defaults.placement_group_strategy, "cluster") - network_interfaces = try(each.value.network_interfaces, var.eks_managed_node_group_defaults.network_interfaces, []) - maintenance_options = try(each.value.maintenance_options, var.eks_managed_node_group_defaults.maintenance_options, {}) - private_dns_name_options = try(each.value.private_dns_name_options, var.eks_managed_node_group_defaults.private_dns_name_options, {}) + create_launch_template = each.value.create_launch_template + use_custom_launch_template = each.value.use_custom_launch_template + launch_template_id = each.value.launch_template_id + launch_template_name = coalesce(each.value.launch_template_name, each.key) + launch_template_use_name_prefix = each.value.launch_template_use_name_prefix + launch_template_version = each.value.launch_template_version + launch_template_default_version = each.value.launch_template_default_version + update_launch_template_default_version = each.value.update_launch_template_default_version + launch_template_description = coalesce(each.value.launch_template_description, "Custom launch template for ${coalesce(each.value.name, each.key)} EKS managed node group") + launch_template_tags = each.value.launch_template_tags + tag_specifications = each.value.tag_specifications + + ebs_optimized = each.value.ebs_optimized + key_name = each.value.key_name + disable_api_termination = each.value.disable_api_termination + kernel_id = each.value.kernel_id + ram_disk_id = each.value.ram_disk_id + + block_device_mappings = each.value.block_device_mappings + capacity_reservation_specification = each.value.capacity_reservation_specification + cpu_options = each.value.cpu_options + credit_specification = each.value.credit_specification + enclave_options = each.value.enclave_options + instance_market_options = each.value.instance_market_options + license_specifications = each.value.license_specifications + metadata_options = each.value.metadata_options + enable_monitoring = each.value.enable_monitoring + enable_efa_support = each.value.enable_efa_support + enable_efa_only = each.value.enable_efa_only + efa_indices = each.value.efa_indices + create_placement_group = each.value.create_placement_group + placement = each.value.placement + network_interfaces = each.value.network_interfaces + maintenance_options = each.value.maintenance_options + private_dns_name_options = each.value.private_dns_name_options # IAM role - create_iam_role = try(each.value.create_iam_role, var.eks_managed_node_group_defaults.create_iam_role, true) - iam_role_arn = try(each.value.iam_role_arn, var.eks_managed_node_group_defaults.iam_role_arn, null) - iam_role_name = try(each.value.iam_role_name, var.eks_managed_node_group_defaults.iam_role_name, null) - iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, var.eks_managed_node_group_defaults.iam_role_use_name_prefix, true) - iam_role_path = try(each.value.iam_role_path, var.eks_managed_node_group_defaults.iam_role_path, null) - iam_role_description = try(each.value.iam_role_description, var.eks_managed_node_group_defaults.iam_role_description, "EKS managed node group IAM role") - iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.eks_managed_node_group_defaults.iam_role_permissions_boundary, null) - iam_role_tags = try(each.value.iam_role_tags, var.eks_managed_node_group_defaults.iam_role_tags, {}) - iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.eks_managed_node_group_defaults.iam_role_attach_cni_policy, true) - # To better understand why this `lookup()` logic is required, see: - # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 - iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.eks_managed_node_group_defaults, "iam_role_additional_policies", {})) - create_iam_role_policy = try(each.value.create_iam_role_policy, var.eks_managed_node_group_defaults.create_iam_role_policy, true) - iam_role_policy_statements = try(each.value.iam_role_policy_statements, var.eks_managed_node_group_defaults.iam_role_policy_statements, []) - - # Autoscaling group schedule - create_schedule = try(each.value.create_schedule, var.eks_managed_node_group_defaults.create_schedule, true) - schedules = try(each.value.schedules, var.eks_managed_node_group_defaults.schedules, {}) + create_iam_role = each.value.create_iam_role + iam_role_arn = each.value.iam_role_arn + iam_role_name = each.value.iam_role_name + iam_role_use_name_prefix = each.value.iam_role_use_name_prefix + iam_role_path = each.value.iam_role_path + iam_role_description = each.value.iam_role_description + iam_role_permissions_boundary = each.value.iam_role_permissions_boundary + iam_role_tags = each.value.iam_role_tags + iam_role_attach_cni_policy = each.value.iam_role_attach_cni_policy + iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", null) + create_iam_role_policy = each.value.create_iam_role_policy + iam_role_policy_statements = each.value.iam_role_policy_statements # Security group - vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.eks_managed_node_group_defaults.vpc_security_group_ids, []))) - cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, var.eks_managed_node_group_defaults.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null + vpc_security_group_ids = compact(concat([local.node_security_group_id], each.value.vpc_security_group_ids)) + cluster_primary_security_group_id = each.value.attach_cluster_primary_security_group ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null + create_security_group = each.value.create_security_group + security_group_name = each.value.security_group_name + security_group_use_name_prefix = each.value.security_group_use_name_prefix + security_group_description = each.value.security_group_description + security_group_ingress_rules = each.value.security_group_ingress_rules + security_group_egress_rules = each.value.security_group_egress_rules + security_group_tags = each.value.security_group_tags - tags = merge(var.tags, try(each.value.tags, var.eks_managed_node_group_defaults.tags, {})) + tags = merge( + var.tags, + each.value.tags, + ) } ################################################################################ @@ -419,150 +390,146 @@ module "eks_managed_node_group" { module "self_managed_node_group" { source = "./modules/self-managed-node-group" - for_each = { for k, v in var.self_managed_node_groups : k => v if var.create } + for_each = var.create && var.self_managed_node_groups != null ? var.self_managed_node_groups : {} + + create = each.value.create + + region = var.region - create = try(each.value.create, true) + # Pass through values to reduce GET requests from data sources + partition = local.partition + account_id = local.account_id - cluster_name = time_sleep.this[0].triggers["cluster_name"] + cluster_name = time_sleep.this[0].triggers["name"] # Autoscaling Group - create_autoscaling_group = try(each.value.create_autoscaling_group, var.self_managed_node_group_defaults.create_autoscaling_group, true) - - name = try(each.value.name, each.key) - use_name_prefix = try(each.value.use_name_prefix, var.self_managed_node_group_defaults.use_name_prefix, true) - - availability_zones = try(each.value.availability_zones, var.self_managed_node_group_defaults.availability_zones, null) - subnet_ids = try(each.value.subnet_ids, var.self_managed_node_group_defaults.subnet_ids, var.subnet_ids) - - min_size = try(each.value.min_size, var.self_managed_node_group_defaults.min_size, 0) - max_size = try(each.value.max_size, var.self_managed_node_group_defaults.max_size, 3) - desired_size = try(each.value.desired_size, var.self_managed_node_group_defaults.desired_size, 1) - desired_size_type = try(each.value.desired_size_type, var.self_managed_node_group_defaults.desired_size_type, null) - capacity_rebalance = try(each.value.capacity_rebalance, var.self_managed_node_group_defaults.capacity_rebalance, null) - min_elb_capacity = try(each.value.min_elb_capacity, var.self_managed_node_group_defaults.min_elb_capacity, null) - wait_for_elb_capacity = try(each.value.wait_for_elb_capacity, var.self_managed_node_group_defaults.wait_for_elb_capacity, null) - wait_for_capacity_timeout = try(each.value.wait_for_capacity_timeout, var.self_managed_node_group_defaults.wait_for_capacity_timeout, null) - default_cooldown = try(each.value.default_cooldown, var.self_managed_node_group_defaults.default_cooldown, null) - default_instance_warmup = try(each.value.default_instance_warmup, var.self_managed_node_group_defaults.default_instance_warmup, null) - protect_from_scale_in = try(each.value.protect_from_scale_in, var.self_managed_node_group_defaults.protect_from_scale_in, null) - context = try(each.value.context, var.self_managed_node_group_defaults.context, null) - - target_group_arns = try(each.value.target_group_arns, var.self_managed_node_group_defaults.target_group_arns, []) - create_placement_group = try(each.value.create_placement_group, var.self_managed_node_group_defaults.create_placement_group, false) - placement_group = try(each.value.placement_group, var.self_managed_node_group_defaults.placement_group, null) - placement_group_az = try(each.value.placement_group_az, var.self_managed_node_group_defaults.placement_group_az, null) - health_check_type = try(each.value.health_check_type, var.self_managed_node_group_defaults.health_check_type, null) - health_check_grace_period = try(each.value.health_check_grace_period, var.self_managed_node_group_defaults.health_check_grace_period, null) - - ignore_failed_scaling_activities = try(each.value.ignore_failed_scaling_activities, var.self_managed_node_group_defaults.ignore_failed_scaling_activities, null) - - force_delete = try(each.value.force_delete, var.self_managed_node_group_defaults.force_delete, null) - force_delete_warm_pool = try(each.value.force_delete_warm_pool, var.self_managed_node_group_defaults.force_delete_warm_pool, null) - termination_policies = try(each.value.termination_policies, var.self_managed_node_group_defaults.termination_policies, []) - suspended_processes = try(each.value.suspended_processes, var.self_managed_node_group_defaults.suspended_processes, []) - max_instance_lifetime = try(each.value.max_instance_lifetime, var.self_managed_node_group_defaults.max_instance_lifetime, null) - - enabled_metrics = try(each.value.enabled_metrics, var.self_managed_node_group_defaults.enabled_metrics, []) - metrics_granularity = try(each.value.metrics_granularity, var.self_managed_node_group_defaults.metrics_granularity, null) - service_linked_role_arn = try(each.value.service_linked_role_arn, var.self_managed_node_group_defaults.service_linked_role_arn, null) - - initial_lifecycle_hooks = try(each.value.initial_lifecycle_hooks, var.self_managed_node_group_defaults.initial_lifecycle_hooks, []) - instance_maintenance_policy = try(each.value.instance_maintenance_policy, var.self_managed_node_group_defaults.instance_maintenance_policy, {}) - instance_refresh = try(each.value.instance_refresh, var.self_managed_node_group_defaults.instance_refresh, local.default_instance_refresh) - use_mixed_instances_policy = try(each.value.use_mixed_instances_policy, var.self_managed_node_group_defaults.use_mixed_instances_policy, false) - mixed_instances_policy = try(each.value.mixed_instances_policy, var.self_managed_node_group_defaults.mixed_instances_policy, null) - warm_pool = try(each.value.warm_pool, var.self_managed_node_group_defaults.warm_pool, {}) - - delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) - autoscaling_group_tags = try(each.value.autoscaling_group_tags, var.self_managed_node_group_defaults.autoscaling_group_tags, {}) + create_autoscaling_group = each.value.create_autoscaling_group + + name = coalesce(each.value.name, each.key) + use_name_prefix = each.value.use_name_prefix + + availability_zones = each.value.availability_zones + subnet_ids = coalesce(each.value.subnet_ids, var.subnet_ids) + + min_size = each.value.min_size + max_size = each.value.max_size + desired_size = each.value.desired_size + desired_size_type = each.value.desired_size_type + capacity_rebalance = each.value.capacity_rebalance + default_instance_warmup = each.value.default_instance_warmup + protect_from_scale_in = each.value.protect_from_scale_in + context = each.value.context + + create_placement_group = each.value.create_placement_group + placement_group = each.value.placement_group + health_check_type = each.value.health_check_type + health_check_grace_period = each.value.health_check_grace_period + + ignore_failed_scaling_activities = each.value.ignore_failed_scaling_activities + + force_delete = each.value.force_delete + termination_policies = each.value.termination_policies + suspended_processes = each.value.suspended_processes + max_instance_lifetime = each.value.max_instance_lifetime + + enabled_metrics = each.value.enabled_metrics + metrics_granularity = each.value.metrics_granularity + + initial_lifecycle_hooks = each.value.initial_lifecycle_hooks + instance_maintenance_policy = each.value.instance_maintenance_policy + instance_refresh = each.value.instance_refresh + use_mixed_instances_policy = each.value.use_mixed_instances_policy + mixed_instances_policy = each.value.mixed_instances_policy + + timeouts = each.value.timeouts + autoscaling_group_tags = each.value.autoscaling_group_tags # User data - platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, null) - # TODO - update this when `var.platform` is removed in v21.0 - ami_type = try(each.value.ami_type, var.self_managed_node_group_defaults.ami_type, "AL2_x86_64") - cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") - cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") - cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") - additional_cluster_dns_ips = try(each.value.additional_cluster_dns_ips, var.self_managed_node_group_defaults.additional_cluster_dns_ips, []) - cluster_ip_family = var.cluster_ip_family - pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.self_managed_node_group_defaults.pre_bootstrap_user_data, "") - post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "") - bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "") - user_data_template_path = try(each.value.user_data_template_path, var.self_managed_node_group_defaults.user_data_template_path, "") - cloudinit_pre_nodeadm = try(each.value.cloudinit_pre_nodeadm, var.self_managed_node_group_defaults.cloudinit_pre_nodeadm, []) - cloudinit_post_nodeadm = try(each.value.cloudinit_post_nodeadm, var.self_managed_node_group_defaults.cloudinit_post_nodeadm, []) + ami_type = try(each.value.ami_type, null) + cluster_endpoint = try(time_sleep.this[0].triggers["endpoint"], "") + cluster_auth_base64 = try(time_sleep.this[0].triggers["certificate_authority_data"], "") + cluster_service_cidr = try(time_sleep.this[0].triggers["service_cidr"], "") + additional_cluster_dns_ips = try(each.value.additional_cluster_dns_ips, null) + cluster_ip_family = var.ip_family + pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, null) + post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, null) + bootstrap_extra_args = try(each.value.bootstrap_extra_args, null) + user_data_template_path = try(each.value.user_data_template_path, null) + cloudinit_pre_nodeadm = try(each.value.cloudinit_pre_nodeadm, null) + cloudinit_post_nodeadm = try(each.value.cloudinit_post_nodeadm, null) # Launch Template - create_launch_template = try(each.value.create_launch_template, var.self_managed_node_group_defaults.create_launch_template, true) - launch_template_id = try(each.value.launch_template_id, var.self_managed_node_group_defaults.launch_template_id, "") - launch_template_name = try(each.value.launch_template_name, var.self_managed_node_group_defaults.launch_template_name, each.key) - launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.self_managed_node_group_defaults.launch_template_use_name_prefix, true) - launch_template_version = try(each.value.launch_template_version, var.self_managed_node_group_defaults.launch_template_version, null) - launch_template_default_version = try(each.value.launch_template_default_version, var.self_managed_node_group_defaults.launch_template_default_version, null) - update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.self_managed_node_group_defaults.update_launch_template_default_version, true) - launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") - launch_template_tags = try(each.value.launch_template_tags, var.self_managed_node_group_defaults.launch_template_tags, {}) - tag_specifications = try(each.value.tag_specifications, var.self_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface"]) - - ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) - ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") - cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"]) - instance_type = try(each.value.instance_type, var.self_managed_node_group_defaults.instance_type, "m6i.large") - key_name = try(each.value.key_name, var.self_managed_node_group_defaults.key_name, null) - - disable_api_termination = try(each.value.disable_api_termination, var.self_managed_node_group_defaults.disable_api_termination, null) - instance_initiated_shutdown_behavior = try(each.value.instance_initiated_shutdown_behavior, var.self_managed_node_group_defaults.instance_initiated_shutdown_behavior, null) - kernel_id = try(each.value.kernel_id, var.self_managed_node_group_defaults.kernel_id, null) - ram_disk_id = try(each.value.ram_disk_id, var.self_managed_node_group_defaults.ram_disk_id, null) - - block_device_mappings = try(each.value.block_device_mappings, var.self_managed_node_group_defaults.block_device_mappings, {}) - capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.self_managed_node_group_defaults.capacity_reservation_specification, {}) - cpu_options = try(each.value.cpu_options, var.self_managed_node_group_defaults.cpu_options, {}) - credit_specification = try(each.value.credit_specification, var.self_managed_node_group_defaults.credit_specification, {}) - elastic_gpu_specifications = try(each.value.elastic_gpu_specifications, var.self_managed_node_group_defaults.elastic_gpu_specifications, {}) - elastic_inference_accelerator = try(each.value.elastic_inference_accelerator, var.self_managed_node_group_defaults.elastic_inference_accelerator, {}) - enclave_options = try(each.value.enclave_options, var.self_managed_node_group_defaults.enclave_options, {}) - hibernation_options = try(each.value.hibernation_options, var.self_managed_node_group_defaults.hibernation_options, {}) - instance_requirements = try(each.value.instance_requirements, var.self_managed_node_group_defaults.instance_requirements, {}) - instance_market_options = try(each.value.instance_market_options, var.self_managed_node_group_defaults.instance_market_options, {}) - license_specifications = try(each.value.license_specifications, var.self_managed_node_group_defaults.license_specifications, {}) - metadata_options = try(each.value.metadata_options, var.self_managed_node_group_defaults.metadata_options, local.metadata_options) - enable_monitoring = try(each.value.enable_monitoring, var.self_managed_node_group_defaults.enable_monitoring, true) - enable_efa_support = try(each.value.enable_efa_support, var.self_managed_node_group_defaults.enable_efa_support, false) - enable_efa_only = try(each.value.enable_efa_only, var.self_managed_node_group_defaults.enable_efa_only, false) - efa_indices = try(each.value.efa_indices, var.self_managed_node_group_defaults.efa_indices, [0]) - network_interfaces = try(each.value.network_interfaces, var.self_managed_node_group_defaults.network_interfaces, []) - placement = try(each.value.placement, var.self_managed_node_group_defaults.placement, {}) - maintenance_options = try(each.value.maintenance_options, var.self_managed_node_group_defaults.maintenance_options, {}) - private_dns_name_options = try(each.value.private_dns_name_options, var.self_managed_node_group_defaults.private_dns_name_options, {}) + create_launch_template = try(each.value.create_launch_template, null) + launch_template_id = try(each.value.launch_template_id, null) + launch_template_name = coalesce(each.value.launch_template_name, each.key) + launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, null) + launch_template_version = try(each.value.launch_template_version, null) + launch_template_default_version = try(each.value.launch_template_default_version, null) + update_launch_template_default_version = try(each.value.update_launch_template_default_version, null) + launch_template_description = coalesce(each.value.launch_template_description, "Custom launch template for ${coalesce(each.value.name, each.key)} self managed node group") + launch_template_tags = try(each.value.launch_template_tags, null) + tag_specifications = try(each.value.tag_specifications, null) + + ebs_optimized = try(each.value.ebs_optimized, null) + ami_id = try(each.value.ami_id, null) + kubernetes_version = try(each.value.kubernetes_version, time_sleep.this[0].triggers["kubernetes_version"]) + instance_type = try(each.value.instance_type, null) + key_name = try(each.value.key_name, null) + + disable_api_termination = try(each.value.disable_api_termination, null) + instance_initiated_shutdown_behavior = try(each.value.instance_initiated_shutdown_behavior, null) + kernel_id = try(each.value.kernel_id, null) + ram_disk_id = try(each.value.ram_disk_id, null) + + block_device_mappings = try(each.value.block_device_mappings, null) + capacity_reservation_specification = try(each.value.capacity_reservation_specification, null) + cpu_options = try(each.value.cpu_options, null) + credit_specification = try(each.value.credit_specification, null) + enclave_options = try(each.value.enclave_options, null) + instance_requirements = try(each.value.instance_requirements, null) + instance_market_options = try(each.value.instance_market_options, null) + license_specifications = try(each.value.license_specifications, null) + metadata_options = try(each.value.metadata_options, null) + enable_monitoring = try(each.value.enable_monitoring, null) + enable_efa_support = try(each.value.enable_efa_support, null) + enable_efa_only = try(each.value.enable_efa_only, null) + efa_indices = try(each.value.efa_indices, null) + network_interfaces = try(each.value.network_interfaces, null) + placement = try(each.value.placement, null) + maintenance_options = try(each.value.maintenance_options, null) + private_dns_name_options = try(each.value.private_dns_name_options, null) # IAM role - create_iam_instance_profile = try(each.value.create_iam_instance_profile, var.self_managed_node_group_defaults.create_iam_instance_profile, true) - iam_instance_profile_arn = try(each.value.iam_instance_profile_arn, var.self_managed_node_group_defaults.iam_instance_profile_arn, null) - iam_role_name = try(each.value.iam_role_name, var.self_managed_node_group_defaults.iam_role_name, null) - iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, var.self_managed_node_group_defaults.iam_role_use_name_prefix, true) - iam_role_path = try(each.value.iam_role_path, var.self_managed_node_group_defaults.iam_role_path, null) - iam_role_description = try(each.value.iam_role_description, var.self_managed_node_group_defaults.iam_role_description, "Self managed node group IAM role") - iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.self_managed_node_group_defaults.iam_role_permissions_boundary, null) - iam_role_tags = try(each.value.iam_role_tags, var.self_managed_node_group_defaults.iam_role_tags, {}) - iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.self_managed_node_group_defaults.iam_role_attach_cni_policy, true) - # To better understand why this `lookup()` logic is required, see: - # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 - iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.self_managed_node_group_defaults, "iam_role_additional_policies", {})) - create_iam_role_policy = try(each.value.create_iam_role_policy, var.self_managed_node_group_defaults.create_iam_role_policy, true) - iam_role_policy_statements = try(each.value.iam_role_policy_statements, var.self_managed_node_group_defaults.iam_role_policy_statements, []) + create_iam_instance_profile = try(each.value.create_iam_instance_profile, null) + iam_instance_profile_arn = try(each.value.iam_instance_profile_arn, null) + iam_role_name = try(each.value.iam_role_name, null) + iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, true) + iam_role_path = try(each.value.iam_role_path, null) + iam_role_description = try(each.value.iam_role_description, null) + iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, null) + iam_role_tags = try(each.value.iam_role_tags, null) + iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, null) + iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", null) + create_iam_role_policy = try(each.value.create_iam_role_policy, null) + iam_role_policy_statements = try(each.value.iam_role_policy_statements, null) # Access entry - create_access_entry = try(each.value.create_access_entry, var.self_managed_node_group_defaults.create_access_entry, true) - iam_role_arn = try(each.value.iam_role_arn, var.self_managed_node_group_defaults.iam_role_arn, null) - - # Autoscaling group schedule - create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, true) - schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, {}) + create_access_entry = try(each.value.create_access_entry, null) + iam_role_arn = try(each.value.iam_role_arn, null) # Security group - vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.self_managed_node_group_defaults.vpc_security_group_ids, []))) - cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, var.self_managed_node_group_defaults.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null + vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, []))) + cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null + create_security_group = try(each.value.create_security_group, null) + security_group_name = try(each.value.security_group_name, null) + security_group_use_name_prefix = try(each.value.security_group_use_name_prefix, null) + security_group_description = try(each.value.security_group_description, null) + security_group_ingress_rules = try(each.value.security_group_ingress_rules, null) + security_group_egress_rules = try(each.value.security_group_egress_rules, null) + security_group_tags = try(each.value.security_group_tags, null) - tags = merge(var.tags, try(each.value.tags, var.self_managed_node_group_defaults.tags, {})) + tags = merge( + var.tags, + each.value.tags, + ) } diff --git a/outputs.tf b/outputs.tf index 688076f604..95afd4365b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,4 +1,5 @@ locals { + # https://github.com/aws/containers-roadmap/issues/2038#issuecomment-2278450601 dualstack_oidc_issuer_url = try(replace(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://oidc.eks.", "https://oidc-eks."), ".amazonaws.com/", ".api.aws/"), null) } @@ -83,7 +84,7 @@ output "cluster_primary_security_group_id" { output "cluster_service_cidr" { description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" - value = var.cluster_ip_family == "ipv6" ? try(aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv6_cidr, null) : try(aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv4_cidr, null) + value = var.ip_family == "ipv6" ? try(aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv6_cidr, null) : try(aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv4_cidr, null) } output "cluster_ip_family" { diff --git a/templates/linux_user_data.tpl b/templates/al2_user_data.tpl similarity index 100% rename from templates/linux_user_data.tpl rename to templates/al2_user_data.tpl diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index 419f601de8..2696041cea 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -17,14 +17,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws](#provider\_aws) | >= 6.0 | ## Modules @@ -33,7 +33,7 @@ Note that this example may create resources which cost money. Run `terraform des | [disabled\_fargate\_profile](#module\_disabled\_fargate\_profile) | ../../modules/fargate-profile | n/a | | [eks](#module\_eks) | ../.. | n/a | | [fargate\_profile](#module\_fargate\_profile) | ../../modules/fargate-profile | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/tests/eks-fargate-profile/main.tf b/tests/eks-fargate-profile/main.tf index bf91d522a8..795e736fc7 100644 --- a/tests/eks-fargate-profile/main.tf +++ b/tests/eks-fargate-profile/main.tf @@ -11,9 +11,9 @@ data "aws_availability_zones" "available" { } locals { - name = "ex-${basename(path.cwd)}" - cluster_version = "1.33" - region = "eu-west-1" + name = "ex-${basename(path.cwd)}" + kubernetes_version = "1.33" + region = "eu-west-1" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -32,11 +32,11 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_public_access = true + name = local.name + kubernetes_version = local.kubernetes_version + endpoint_public_access = true - cluster_addons = { + addons = { kube-proxy = {} vpc-cni = {} coredns = { @@ -51,14 +51,8 @@ module "eks" { control_plane_subnet_ids = module.vpc.intra_subnets # Fargate profiles use the cluster primary security group so these are not utilized - create_cluster_security_group = false - create_node_security_group = false - - fargate_profile_defaults = { - iam_role_additional_policies = { - additional = aws_iam_policy.additional.arn - } - } + create_security_group = false + create_node_security_group = false fargate_profiles = { example = { @@ -78,6 +72,10 @@ module "eks" { } ] + iam_role_additional_policies = { + additional = aws_iam_policy.additional.arn + } + # Using specific subnets instead of the subnets supplied for the cluster itself subnet_ids = [module.vpc.private_subnets[1]] @@ -125,7 +123,7 @@ module "disabled_fargate_profile" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index 67bc82c7a9..efe26912ce 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -17,8 +17,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index 20514e1bdd..3c2ec900b1 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 96a5011da3..15a05fa9e6 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -17,14 +17,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws](#provider\_aws) | >= 6.0 | ## Modules @@ -33,11 +33,11 @@ Note that this example may create resources which cost money. Run `terraform des | [aws\_vpc\_cni\_ipv6\_pod\_identity](#module\_aws\_vpc\_cni\_ipv6\_pod\_identity) | terraform-aws-modules/eks-pod-identity/aws | ~> 1.6 | | [disabled\_eks](#module\_disabled\_eks) | ../.. | n/a | | [disabled\_eks\_managed\_node\_group](#module\_disabled\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | -| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 2.1 | +| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 4.0 | | [eks](#module\_eks) | ../.. | n/a | | [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index 010a55fa1b..7d8adc08e3 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -13,9 +13,9 @@ data "aws_availability_zones" "available" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.33" - region = "eu-west-1" + name = "ex-${replace(basename(path.cwd), "_", "-")}" + kubernetes_version = "1.33" + region = "eu-west-1" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -34,21 +34,17 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_public_access = true + name = local.name + kubernetes_version = local.kubernetes_version + endpoint_public_access = true # IPV6 - cluster_ip_family = "ipv6" + ip_family = "ipv6" create_cni_ipv6_iam_policy = true enable_cluster_creator_admin_permissions = true - # Enable EFA support by adding necessary security group rules - # to the shared node security group - enable_efa_support = true - - cluster_addons = { + addons = { coredns = { most_recent = true } @@ -79,11 +75,11 @@ module "eks" { } } - cluster_upgrade_policy = { + upgrade_policy = { support_type = "STANDARD" } - cluster_zonal_shift_config = { + zonal_shift_config = { enabled = true } @@ -91,11 +87,6 @@ module "eks" { subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets - eks_managed_node_group_defaults = { - ami_type = "AL2023_x86_64_STANDARD" - instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] - } - eks_managed_node_groups = { # Default node group - as provided by AWS EKS default_node_group = { @@ -114,8 +105,8 @@ module "eks" { placement_group = { create_placement_group = true - # forces the subnet lookup to be restricted to this availability zone - placement_group_az = element(local.azs, 3) + subnet_ids = slice(module.vpc.private_subnets, 0, 1) + instance_types = ["m5.large", "m5n.large", "m5zn.large"] } # AL2023 node group utilizing new user data format which utilizes nodeadm @@ -135,8 +126,6 @@ module "eks" { kubelet: config: shutdownGracePeriod: 30s - featureGates: - DisableKubeletCloudCredentialProviders: true EOT } ] @@ -238,8 +227,6 @@ module "eks" { kubelet: config: shutdownGracePeriod: 30s - featureGates: - DisableKubeletCloudCredentialProviders: true EOT content_type = "application/node.eks.aws" }] @@ -359,7 +346,7 @@ module "eks" { # This will: # 1. Create a placement group to place the instances close to one another - # 2. Ignore subnets that reside in AZs that do not support the instance type + # 2. Create and attach the necessary security group rules (and security group) # 3. Expose all of the available EFA interfaces on the launch template enable_efa_support = true enable_efa_only = true @@ -476,7 +463,7 @@ module "disabled_eks_managed_node_group" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr @@ -523,7 +510,7 @@ module "aws_vpc_cni_ipv6_pod_identity" { module "ebs_kms_key" { source = "terraform-aws-modules/kms/aws" - version = "~> 2.1" + version = "~> 4.0" description = "Customer managed key to encrypt EKS managed node group volumes" @@ -605,7 +592,7 @@ data "aws_ami" "eks_default" { filter { name = "name" - values = ["amazon-eks-node-al2023-x86_64-standard-${local.cluster_version}-v*"] + values = ["amazon-eks-node-al2023-x86_64-standard-${local.kubernetes_version}-v*"] } } @@ -615,7 +602,7 @@ data "aws_ami" "eks_default_arm" { filter { name = "name" - values = ["amazon-eks-node-al2023-arm64-standard-${local.cluster_version}-v*"] + values = ["amazon-eks-node-al2023-arm64-standard-${local.kubernetes_version}-v*"] } } @@ -625,7 +612,7 @@ data "aws_ami" "eks_default_bottlerocket" { filter { name = "name" - values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"] + values = ["bottlerocket-aws-k8s-${local.kubernetes_version}-x86_64-*"] } } diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/tests/fast-addons/README.md b/tests/fast-addons/README.md deleted file mode 100644 index cd1be0dcd6..0000000000 --- a/tests/fast-addons/README.md +++ /dev/null @@ -1,92 +0,0 @@ -# Fast Addons - -Refer to https://github.com/terraform-aws-modules/terraform-aws-eks/pull/3214 for additional information. - - - -## Usage - -To provision the provided configurations you need to execute: - -```bash -$ terraform init -$ terraform plan -$ terraform apply --auto-approve -``` - -Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | ../.. | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_route_table_association.custom_network](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association) | resource | -| [aws_subnet.custom_network](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource | -| [aws_vpc_ipv4_cidr_block_association.custom_network](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_ipv4_cidr_block_association) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | - -## Inputs - -No inputs. - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes | -| [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created | -| [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created | -| [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | -| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | -| [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | -| [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | -| [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | -| [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | -| [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | -| [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts | -| [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | -| [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) | -| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster | -| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | -| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | -| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | -| [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | -| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group | -| [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from | -| [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` | -| [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | -| [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | -| [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | -| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | -| [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key | -| [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key | -| [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key | -| [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | -| [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | -| [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | -| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | -| [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | -| [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | - diff --git a/tests/fast-addons/main.tf b/tests/fast-addons/main.tf deleted file mode 100644 index e7a5b9cc55..0000000000 --- a/tests/fast-addons/main.tf +++ /dev/null @@ -1,159 +0,0 @@ -provider "aws" { - region = local.region -} - -locals { - name = "ex-${basename(path.cwd)}" - cluster_version = "1.33" - region = "eu-west-1" - - tags = { - Test = local.name - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } -} - -################################################################################ -# EKS Module -################################################################################ - -module "eks" { - source = "../.." - - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_public_access = true - - enable_cluster_creator_admin_permissions = true - - # Disable the default self-managed addons to avoid the penalty of adopting them later - bootstrap_self_managed_addons = false - - # Addons will be provisioned net new via the EKS addon API - cluster_addons = { - coredns = { - most_recent = true - } - eks-pod-identity-agent = { - before_compute = true - most_recent = true - } - kube-proxy = { - most_recent = true - } - vpc-cni = { - most_recent = true - before_compute = true - configuration_values = jsonencode({ - env = { - # Use subnet tags to avoid the need to inject the ENIConfig - # which requires a live API server endpoint which leads to a dependency of: - # Control plane -> API request to create ENIConfig -> VPC CNI addon -> nodes/compute - # With the subnet discovery feature, we can avoid this dependency: - # Control plane -> VPC CNI addon -> nodes/compute - ENABLE_SUBNET_DISCOVERY = "true" - } - }) - } - } - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - eks_managed_node_groups = { - example = { - instance_types = ["m6i.large"] - - min_size = 2 - max_size = 5 - desired_size = 2 - } - } - - tags = local.tags -} - -################################################################################ -# VPC -################################################################################ - -data "aws_availability_zones" "available" { - # Exclude local zones - filter { - name = "opt-in-status" - values = ["opt-in-not-required"] - } -} - -locals { - vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) -} - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" - - name = local.name - cidr = local.vpc_cidr - - azs = local.azs - private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] - public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] - - enable_nat_gateway = true - single_nat_gateway = true - - public_subnet_tags = { - "kubernetes.io/role/elb" = 1 - } - - tags = local.tags -} - -################################################################################ -# Custom Networking -################################################################################ - -locals { - custom_network_vpc_cidr = "10.99.0.0/16" - - custom_network_subnets = [for k, v in local.azs : cidrsubnet(local.custom_network_vpc_cidr, 4, k)] -} - -resource "aws_vpc_ipv4_cidr_block_association" "custom_network" { - vpc_id = module.vpc.vpc_id - cidr_block = local.custom_network_vpc_cidr -} - -resource "aws_subnet" "custom_network" { - count = length(local.custom_network_subnets) - - vpc_id = module.vpc.vpc_id - cidr_block = element(local.custom_network_subnets, count.index) - - tags = merge( - local.tags, - { - # Tag for subnet discovery - "kubernetes.io/role/cni" = 1 - "kubernetes.io/role/internal-elb" = 1 - } - ) - - depends_on = [ - aws_vpc_ipv4_cidr_block_association.custom_network - ] -} - -resource "aws_route_table_association" "custom_network" { - count = length(local.custom_network_subnets) - - subnet_id = element(aws_subnet.custom_network[*].id, count.index) - route_table_id = element(module.vpc.private_route_table_ids, 0) - - depends_on = [ - aws_vpc_ipv4_cidr_block_association.custom_network - ] -} diff --git a/tests/fast-addons/outputs.tf b/tests/fast-addons/outputs.tf deleted file mode 100644 index 9357464c29..0000000000 --- a/tests/fast-addons/outputs.tf +++ /dev/null @@ -1,226 +0,0 @@ -################################################################################ -# Cluster -################################################################################ - -output "cluster_arn" { - description = "The Amazon Resource Name (ARN) of the cluster" - value = module.eks.cluster_arn -} - -output "cluster_certificate_authority_data" { - description = "Base64 encoded certificate data required to communicate with the cluster" - value = module.eks.cluster_certificate_authority_data -} - -output "cluster_endpoint" { - description = "Endpoint for your Kubernetes API server" - value = module.eks.cluster_endpoint -} - -output "cluster_id" { - description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" - value = module.eks.cluster_id -} - -output "cluster_name" { - description = "The name of the EKS cluster" - value = module.eks.cluster_name -} - -output "cluster_oidc_issuer_url" { - description = "The URL on the EKS cluster for the OpenID Connect identity provider" - value = module.eks.cluster_oidc_issuer_url -} - -output "cluster_dualstack_oidc_issuer_url" { - description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" - value = module.eks.cluster_dualstack_oidc_issuer_url -} - -output "cluster_platform_version" { - description = "Platform version for the cluster" - value = module.eks.cluster_platform_version -} - -output "cluster_status" { - description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" - value = module.eks.cluster_status -} - -output "cluster_primary_security_group_id" { - description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" - value = module.eks.cluster_primary_security_group_id -} - -output "cluster_service_cidr" { - description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" - value = module.eks.cluster_service_cidr -} - -output "cluster_ip_family" { - description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" - value = module.eks.cluster_ip_family -} - -################################################################################ -# Access Entry -################################################################################ - -output "access_entries" { - description = "Map of access entries created and their attributes" - value = module.eks.access_entries -} - -################################################################################ -# KMS Key -################################################################################ - -output "kms_key_arn" { - description = "The Amazon Resource Name (ARN) of the key" - value = module.eks.kms_key_arn -} - -output "kms_key_id" { - description = "The globally unique identifier for the key" - value = module.eks.kms_key_id -} - -output "kms_key_policy" { - description = "The IAM resource policy set on the key" - value = module.eks.kms_key_policy -} - -################################################################################ -# Security Group -################################################################################ - -output "cluster_security_group_arn" { - description = "Amazon Resource Name (ARN) of the cluster security group" - value = module.eks.cluster_security_group_arn -} - -output "cluster_security_group_id" { - description = "ID of the cluster security group" - value = module.eks.cluster_security_group_id -} - -################################################################################ -# Node Security Group -################################################################################ - -output "node_security_group_arn" { - description = "Amazon Resource Name (ARN) of the node shared security group" - value = module.eks.node_security_group_arn -} - -output "node_security_group_id" { - description = "ID of the node shared security group" - value = module.eks.node_security_group_id -} - -################################################################################ -# IRSA -################################################################################ - -output "oidc_provider" { - description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" - value = module.eks.oidc_provider -} - -output "oidc_provider_arn" { - description = "The ARN of the OIDC Provider if `enable_irsa = true`" - value = module.eks.oidc_provider_arn -} - -output "cluster_tls_certificate_sha1_fingerprint" { - description = "The SHA1 fingerprint of the public key of the cluster's certificate" - value = module.eks.cluster_tls_certificate_sha1_fingerprint -} - -################################################################################ -# IAM Role -################################################################################ - -output "cluster_iam_role_name" { - description = "IAM role name of the EKS cluster" - value = module.eks.cluster_iam_role_name -} - -output "cluster_iam_role_arn" { - description = "IAM role ARN of the EKS cluster" - value = module.eks.cluster_iam_role_arn -} - -output "cluster_iam_role_unique_id" { - description = "Stable and unique string identifying the IAM role" - value = module.eks.cluster_iam_role_unique_id -} - -################################################################################ -# EKS Addons -################################################################################ - -output "cluster_addons" { - description = "Map of attribute maps for all EKS cluster addons enabled" - value = module.eks.cluster_addons -} - -################################################################################ -# EKS Identity Provider -################################################################################ - -output "cluster_identity_providers" { - description = "Map of attribute maps for all EKS identity providers enabled" - value = module.eks.cluster_identity_providers -} - -################################################################################ -# CloudWatch Log Group -################################################################################ - -output "cloudwatch_log_group_name" { - description = "Name of cloudwatch log group created" - value = module.eks.cloudwatch_log_group_name -} - -output "cloudwatch_log_group_arn" { - description = "Arn of cloudwatch log group created" - value = module.eks.cloudwatch_log_group_arn -} - -################################################################################ -# Fargate Profile -################################################################################ - -output "fargate_profiles" { - description = "Map of attribute maps for all EKS Fargate Profiles created" - value = module.eks.fargate_profiles -} - -################################################################################ -# EKS Managed Node Group -################################################################################ - -output "eks_managed_node_groups" { - description = "Map of attribute maps for all EKS managed node groups created" - value = module.eks.eks_managed_node_groups -} - -output "eks_managed_node_groups_autoscaling_group_names" { - description = "List of the autoscaling group names created by EKS managed node groups" - value = module.eks.eks_managed_node_groups_autoscaling_group_names -} - -################################################################################ -# Self Managed Node Group -################################################################################ - -output "self_managed_node_groups" { - description = "Map of attribute maps for all self managed node groups created" - value = module.eks.self_managed_node_groups -} - -output "self_managed_node_groups_autoscaling_group_names" { - description = "List of the autoscaling group names created by self-managed node groups" - value = module.eks.self_managed_node_groups_autoscaling_group_names -} diff --git a/tests/fast-addons/variables.tf b/tests/fast-addons/variables.tf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/fast-addons/versions.tf b/tests/fast-addons/versions.tf deleted file mode 100644 index 7884c4e1b2..0000000000 --- a/tests/fast-addons/versions.tf +++ /dev/null @@ -1,10 +0,0 @@ -terraform { - required_version = ">= 1.3.2" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" - } - } -} diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 655025789b..feee069124 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -17,14 +17,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | -| [aws](#requirement\_aws) | >= 5.95, < 6.0.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.95, < 6.0.0 | +| [aws](#provider\_aws) | >= 6.0 | ## Modules @@ -32,11 +32,11 @@ Note that this example may create resources which cost money. Run `terraform des |------|--------|---------| | [aws\_vpc\_cni\_ipv4\_pod\_identity](#module\_aws\_vpc\_cni\_ipv4\_pod\_identity) | terraform-aws-modules/eks-pod-identity/aws | ~> 1.6 | | [disabled\_self\_managed\_node\_group](#module\_disabled\_self\_managed\_node\_group) | ../../modules/self-managed-node-group | n/a | -| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 2.0 | +| [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 4.0 | | [eks](#module\_eks) | ../.. | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | -| [kms](#module\_kms) | terraform-aws-modules/kms/aws | ~> 2.1 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | ~> 4.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/tests/self-managed-node-group/main.tf b/tests/self-managed-node-group/main.tf index 351985f359..8629039e7d 100644 --- a/tests/self-managed-node-group/main.tf +++ b/tests/self-managed-node-group/main.tf @@ -13,9 +13,9 @@ data "aws_availability_zones" "available" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.33" - region = "eu-west-1" + name = "ex-${replace(basename(path.cwd), "_", "-")}" + kubernetes_version = "1.33" + region = "eu-west-1" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -34,28 +34,26 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_public_access = true + name = local.name + kubernetes_version = local.kubernetes_version + endpoint_public_access = true enable_cluster_creator_admin_permissions = true - # Enable EFA support by adding necessary security group rules - # to the shared node security group - enable_efa_support = true - - cluster_addons = { + addons = { coredns = { most_recent = true } eks-pod-identity-agent = { - most_recent = true + before_compute = true + most_recent = true } kube-proxy = { most_recent = true } vpc-cni = { - most_recent = true + before_compute = true + most_recent = true pod_identity_association = [{ role_arn = module.aws_vpc_cni_ipv4_pod_identity.iam_role_arn service_account = "aws-node" @@ -69,25 +67,23 @@ module "eks" { # External encryption key create_kms_key = false - cluster_encryption_config = { + encryption_config = { resources = ["secrets"] provider_key_arn = module.kms.key_arn } - self_managed_node_group_defaults = { - ami_type = "AL2023_x86_64_STANDARD" - ami_id = data.aws_ami.eks_default.image_id - - # enable discovery of autoscaling groups by cluster-autoscaler - autoscaling_group_tags = { - "k8s.io/cluster-autoscaler/enabled" : true, - "k8s.io/cluster-autoscaler/${local.name}" : "owned", - } - } - self_managed_node_groups = { # Default node group - as provisioned by the module defaults - default_node_group = {} + default_node_group = { + ami_type = "AL2023_x86_64_STANDARD" + ami_id = data.aws_ami.eks_default.image_id + + # enable discovery of autoscaling groups by cluster-autoscaler + autoscaling_group_tags = { + "k8s.io/cluster-autoscaler/enabled" : true, + "k8s.io/cluster-autoscaler/${local.name}" : "owned", + } + } # Bottlerocket node group bottlerocket = { @@ -153,16 +149,18 @@ module "eks" { spot_allocation_strategy = "capacity-optimized" } - override = [ - { - instance_type = "m5.large" - weighted_capacity = "1" - }, - { - instance_type = "m6i.large" - weighted_capacity = "2" - }, - ] + launch_template = { + override = [ + { + instance_type = "m5.large" + weighted_capacity = "1" + }, + { + instance_type = "m6i.large" + weighted_capacity = "2" + }, + ] + } } } @@ -232,8 +230,6 @@ module "eks" { kubelet: config: shutdownGracePeriod: 30s - featureGates: - DisableKubeletCloudCredentialProviders: true EOT content_type = "application/node.eks.aws" }] @@ -263,28 +259,30 @@ module "eks" { } # ASG configuration - override = [ - { - instance_requirements = { - cpu_manufacturers = ["intel"] - instance_generations = ["current", "previous"] - spot_max_price_percentage_over_lowest_price = 100 - - vcpu_count = { - min = 1 + launch_template = { + override = [ + { + instance_requirements = { + cpu_manufacturers = ["intel"] + instance_generations = ["current", "previous"] + spot_max_price_percentage_over_lowest_price = 100 + + vcpu_count = { + min = 1 + } + + allowed_instance_types = ["t*", "m*"] } - - allowed_instance_types = ["t*", "m*"] } - } - ] + ] + } } } metadata_options = { http_endpoint = "enabled" http_tokens = "required" - http_put_response_hop_limit = 2 + http_put_response_hop_limit = 1 instance_metadata_tags = "disabled" } @@ -334,7 +332,7 @@ module "eks" { # This will: # 1. Create a placement group to place the instances close to one another - # 2. Ignore subnets that reside in AZs that do not support the instance type + # 2. Create and attach the necessary security group rules (and security group) # 3. Expose all of the available EFA interfaces on the launch template enable_efa_support = true enable_efa_only = true @@ -378,7 +376,7 @@ module "disabled_self_managed_node_group" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr @@ -420,7 +418,7 @@ data "aws_ami" "eks_default" { filter { name = "name" - values = ["amazon-eks-node-al2023-x86_64-standard-${local.cluster_version}-v*"] + values = ["amazon-eks-node-al2023-x86_64-standard-${local.kubernetes_version}-v*"] } } @@ -430,7 +428,7 @@ data "aws_ami" "eks_default_bottlerocket" { filter { name = "name" - values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"] + values = ["bottlerocket-aws-k8s-${local.kubernetes_version}-x86_64-*"] } } @@ -446,7 +444,7 @@ module "key_pair" { module "ebs_kms_key" { source = "terraform-aws-modules/kms/aws" - version = "~> 2.0" + version = "~> 4.0" description = "Customer managed key to encrypt EKS managed node group volumes" @@ -470,7 +468,7 @@ module "ebs_kms_key" { module "kms" { source = "terraform-aws-modules/kms/aws" - version = "~> 2.1" + version = "~> 4.0" aliases = ["eks/${local.name}"] description = "${local.name} cluster encryption key" diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index 7884c4e1b2..db13b0a8d2 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } } } diff --git a/tests/user-data/README.md b/tests/user-data/README.md index 88988ab2f4..684a3a7648 100644 --- a/tests/user-data/README.md +++ b/tests/user-data/README.md @@ -17,7 +17,7 @@ $ terraform apply --auto-approve | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | | [local](#requirement\_local) | >= 2.4 | ## Providers diff --git a/tests/user-data/main.tf b/tests/user-data/main.tf index 9a55b3cf77..835d9fc347 100644 --- a/tests/user-data/main.tf +++ b/tests/user-data/main.tf @@ -15,13 +15,15 @@ locals { module "eks_mng_al2_disabled" { source = "../../modules/_user_data" - create = false + ami_type = "AL2_x86_64" + create = false } module "eks_mng_al2_no_op" { source = "../../modules/_user_data" # Hard requirement + ami_type = "AL2_x86_64" cluster_service_cidr = local.cluster_service_cidr } @@ -29,6 +31,7 @@ module "eks_mng_al2_additional" { source = "../../modules/_user_data" # Hard requirement + ami_type = "AL2_x86_64" cluster_service_cidr = local.cluster_service_cidr pre_bootstrap_user_data = <<-EOT @@ -39,6 +42,7 @@ module "eks_mng_al2_additional" { module "eks_mng_al2_custom_ami" { source = "../../modules/_user_data" + ami_type = "AL2_x86_64" cluster_name = local.name cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 @@ -60,6 +64,7 @@ module "eks_mng_al2_custom_ami" { module "eks_mng_al2_custom_ami_ipv6" { source = "../../modules/_user_data" + ami_type = "AL2_x86_64" cluster_name = local.name cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 @@ -82,6 +87,7 @@ module "eks_mng_al2_custom_ami_ipv6" { module "eks_mng_al2_custom_template" { source = "../../modules/_user_data" + ami_type = "AL2_x86_64" cluster_name = local.name cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 @@ -364,12 +370,15 @@ module "self_mng_al2_no_op" { is_eks_managed_node_group = false # Hard requirement + ami_type = "AL2_x86_64" cluster_service_cidr = local.cluster_service_cidr } module "self_mng_al2_bootstrap" { source = "../../modules/_user_data" + ami_type = "AL2_x86_64" + enable_bootstrap_user_data = true is_eks_managed_node_group = false @@ -393,6 +402,8 @@ module "self_mng_al2_bootstrap" { module "self_mng_al2_bootstrap_ipv6" { source = "../../modules/_user_data" + ami_type = "AL2_x86_64" + enable_bootstrap_user_data = true is_eks_managed_node_group = false @@ -417,6 +428,8 @@ module "self_mng_al2_bootstrap_ipv6" { module "self_mng_al2_custom_template" { source = "../../modules/_user_data" + ami_type = "AL2_x86_64" + enable_bootstrap_user_data = true is_eks_managed_node_group = false diff --git a/tests/user-data/versions.tf b/tests/user-data/versions.tf index 31969d6c02..0d69a257f0 100644 --- a/tests/user-data/versions.tf +++ b/tests/user-data/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { local = { diff --git a/variables.tf b/variables.tf index b4881b2988..6b3e972a62 100644 --- a/variables.tf +++ b/variables.tf @@ -4,41 +4,47 @@ variable "create" { default = true } -variable "tags" { - description = "A map of tags to add to all resources" - type = map(string) - default = {} -} - variable "prefix_separator" { description = "The separator to use between the prefix and the generated timestamp for resource names" type = string default = "-" } +variable "region" { + description = "Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration" + type = string + default = null +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + ################################################################################ # Cluster ################################################################################ -variable "cluster_name" { +variable "name" { description = "Name of the EKS cluster" type = string default = "" } -variable "cluster_version" { - description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.27`)" +variable "kubernetes_version" { + description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.33`)" type = string default = null } -variable "cluster_enabled_log_types" { +variable "enabled_log_types" { description = "A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html)" type = list(string) default = ["audit", "api", "authenticator"] } -variable "cluster_force_update_version" { +variable "force_update_version" { description = "Force version update by overriding upgrade-blocking readiness checks when updating a cluster" type = bool default = null @@ -50,31 +56,46 @@ variable "authentication_mode" { default = "API_AND_CONFIG_MAP" } -variable "cluster_compute_config" { +variable "compute_config" { description = "Configuration block for the cluster compute configuration" - type = any - default = {} + type = object({ + enabled = optional(bool, false) + node_pools = optional(list(string)) + node_role_arn = optional(string) + }) + default = null } -variable "cluster_upgrade_policy" { +variable "upgrade_policy" { description = "Configuration block for the cluster upgrade policy" - type = any - default = {} + type = object({ + support_type = optional(string) + }) + default = null } -variable "cluster_remote_network_config" { +variable "remote_network_config" { description = "Configuration block for the cluster remote network configuration" - type = any - default = {} -} - -variable "cluster_zonal_shift_config" { + type = object({ + remote_node_networks = object({ + cidrs = optional(list(string)) + }) + remote_pod_networks = optional(object({ + cidrs = optional(list(string)) + })) + }) + default = null +} + +variable "zonal_shift_config" { description = "Configuration block for the cluster zonal shift" - type = any - default = {} + type = object({ + enabled = optional(bool) + }) + default = null } -variable "cluster_additional_security_group_ids" { +variable "additional_security_group_ids" { description = "List of additional, externally created security group IDs to attach to the cluster control plane" type = list(string) default = [] @@ -92,37 +113,37 @@ variable "subnet_ids" { default = [] } -variable "cluster_endpoint_private_access" { +variable "endpoint_private_access" { description = "Indicates whether or not the Amazon EKS private API server endpoint is enabled" type = bool default = true } -variable "cluster_endpoint_public_access" { +variable "endpoint_public_access" { description = "Indicates whether or not the Amazon EKS public API server endpoint is enabled" type = bool default = false } -variable "cluster_endpoint_public_access_cidrs" { +variable "endpoint_public_access_cidrs" { description = "List of CIDR blocks which can access the Amazon EKS public API server endpoint" type = list(string) default = ["0.0.0.0/0"] } -variable "cluster_ip_family" { +variable "ip_family" { description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created" type = string default = "ipv4" } -variable "cluster_service_ipv4_cidr" { +variable "service_ipv4_cidr" { description = "The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" type = string default = null } -variable "cluster_service_ipv6_cidr" { +variable "service_ipv6_cidr" { description = "The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster" type = string default = null @@ -130,19 +151,26 @@ variable "cluster_service_ipv6_cidr" { variable "outpost_config" { description = "Configuration for the AWS Outpost to provision the cluster on" - type = any - default = {} -} - -variable "cluster_encryption_config" { - description = "Configuration block with encryption configuration for the cluster. To disable secret encryption, set this value to `{}`" - type = any - default = { - resources = ["secrets"] - } -} - -variable "attach_cluster_encryption_policy" { + type = object({ + control_plane_instance_type = optional(string) + control_plane_placement = optional(object({ + group_name = string + })) + outpost_arns = list(string) + }) + default = null +} + +variable "encryption_config" { + description = "Configuration block with encryption configuration for the cluster" + type = object({ + provider_key_arn = optional(string) + resources = optional(list(string), ["secrets"]) + }) + default = {} +} + +variable "attach_encryption_policy" { description = "Indicates whether or not to attach an additional policy for the cluster IAM role to utilize the encryption key provided" type = bool default = true @@ -154,23 +182,20 @@ variable "cluster_tags" { default = {} } -variable "create_cluster_primary_security_group_tags" { +variable "create_primary_security_group_tags" { description = "Indicates whether or not to tag the cluster's primary security group. This security group is created by the EKS service, not the module, and therefore tagging is handled after cluster creation" type = bool default = true } -variable "cluster_timeouts" { +variable "timeouts" { description = "Create, update, and delete timeout configurations for the cluster" - type = map(string) - default = {} -} - -# TODO - hard code to false on next breaking change -variable "bootstrap_self_managed_addons" { - description = "Indicates whether or not to bootstrap self-managed addons after the cluster has been created" - type = bool - default = null + type = object({ + create = optional(string) + update = optional(string) + delete = optional(string) + }) + default = null } ################################################################################ @@ -179,8 +204,23 @@ variable "bootstrap_self_managed_addons" { variable "access_entries" { description = "Map of access entries to add to the cluster" - type = any - default = {} + type = map(object({ + # Access entry + kubernetes_groups = optional(list(string)) + principal_arn = string + type = optional(string, "STANDARD") + user_name = optional(string) + tags = optional(map(string), {}) + # Access policy association + policy_associations = optional(map(object({ + policy_arn = string + access_scope = object({ + namespaces = optional(list(string)) + type = string + }) + }))) + })) + default = {} } variable "enable_cluster_creator_admin_permissions" { @@ -303,13 +343,13 @@ variable "cloudwatch_log_group_tags" { # Cluster Security Group ################################################################################ -variable "create_cluster_security_group" { +variable "create_security_group" { description = "Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default" type = bool default = true } -variable "cluster_security_group_id" { +variable "security_group_id" { description = "Existing security group ID to be attached to the cluster" type = string default = "" @@ -321,31 +361,43 @@ variable "vpc_id" { default = null } -variable "cluster_security_group_name" { +variable "security_group_name" { description = "Name to use on cluster security group created" type = string default = null } -variable "cluster_security_group_use_name_prefix" { +variable "security_group_use_name_prefix" { description = "Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix" type = bool default = true } -variable "cluster_security_group_description" { +variable "security_group_description" { description = "Description of the cluster security group created" type = string default = "EKS cluster security group" } -variable "cluster_security_group_additional_rules" { +variable "security_group_additional_rules" { description = "List of additional security group rules to add to the cluster security group created. Set `source_node_security_group = true` inside rules to set the `node_security_group` as source" - type = any - default = {} -} - -variable "cluster_security_group_tags" { + type = map(object({ + protocol = optional(string, "tcp") + from_port = number + to_port = number + type = optional(string, "ingress") + description = optional(string) + cidr_blocks = optional(list(string)) + ipv6_cidr_blocks = optional(list(string)) + prefix_list_ids = optional(list(string)) + self = optional(bool) + source_node_security_group = optional(bool, false) + source_security_group_id = optional(string) + })) + default = {} +} + +variable "security_group_tags" { description = "A map of additional tags to add to the cluster security group created" type = map(string) default = {} @@ -397,8 +449,20 @@ variable "node_security_group_description" { variable "node_security_group_additional_rules" { description = "List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source" - type = any - default = {} + type = map(object({ + protocol = optional(string, "tcp") + from_port = number + to_port = number + type = optional(string, "ingress") + description = optional(string) + cidr_blocks = optional(list(string)) + ipv6_cidr_blocks = optional(list(string)) + prefix_list_ids = optional(list(string)) + self = optional(bool) + source_cluster_security_group = optional(bool, false) + source_security_group_id = optional(string) + })) + default = {} } variable "node_security_group_enable_recommended_rules" { @@ -413,12 +477,6 @@ variable "node_security_group_tags" { default = {} } -variable "enable_efa_support" { - description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" - type = bool - default = false -} - ################################################################################ # IRSA ################################################################################ @@ -499,44 +557,37 @@ variable "iam_role_additional_policies" { default = {} } -# TODO - will be removed in next breaking change; user can add the policy on their own when needed -variable "enable_security_groups_for_pods" { - description = "Determines whether to add the necessary IAM permission policy for security groups for pods" - type = bool - default = true -} - variable "iam_role_tags" { description = "A map of additional tags to add to the IAM role created" type = map(string) default = {} } -variable "cluster_encryption_policy_use_name_prefix" { +variable "encryption_policy_use_name_prefix" { description = "Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix" type = bool default = true } -variable "cluster_encryption_policy_name" { +variable "encryption_policy_name" { description = "Name to use on cluster encryption policy created" type = string default = null } -variable "cluster_encryption_policy_description" { +variable "encryption_policy_description" { description = "Description of the cluster encryption policy created" type = string default = "Cluster encryption policy to allow cluster role to utilize CMK provided" } -variable "cluster_encryption_policy_path" { +variable "encryption_policy_path" { description = "Cluster encryption policy path" type = string default = null } -variable "cluster_encryption_policy_tags" { +variable "encryption_policy_tags" { description = "A map of additional tags to add to the cluster encryption policy created" type = map(string) default = {} @@ -558,26 +609,59 @@ variable "enable_auto_mode_custom_tags" { # EKS Addons ################################################################################ -variable "cluster_addons" { +variable "addons" { description = "Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name`" - type = any - default = {} -} - -variable "cluster_addons_timeouts" { + type = map(object({ + name = optional(string) # will fall back to map key + before_compute = optional(bool, false) + most_recent = optional(bool, true) + addon_version = optional(string) + configuration_values = optional(string) + pod_identity_association = optional(list(object({ + role_arn = string + service_account = string + }))) + preserve = optional(bool, true) + resolve_conflicts_on_create = optional(string, "NONE") + resolve_conflicts_on_update = optional(string, "OVERWRITE") + service_account_role_arn = optional(string) + timeouts = optional(object({ + create = optional(string) + update = optional(string) + delete = optional(string) + })) + tags = optional(map(string), {}) + })) + default = null +} + +variable "addons_timeouts" { description = "Create, update, and delete timeout configurations for the cluster addons" - type = map(string) - default = {} + type = object({ + create = optional(string) + update = optional(string) + delete = optional(string) + }) + default = null } ################################################################################ # EKS Identity Provider ################################################################################ -variable "cluster_identity_providers" { +variable "identity_providers" { description = "Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA" - type = any - default = {} + type = map(object({ + client_id = string + groups_claim = optional(string) + groups_prefix = optional(string) + identity_provider_config_name = optional(string) # will fall back to map key + issuer_url = string + required_claims = optional(map(string)) + username_claim = optional(string) + username_prefix = optional(string) + })) + default = null } ################################################################################ @@ -638,14 +722,57 @@ variable "node_iam_role_tags" { variable "fargate_profiles" { description = "Map of Fargate Profile definitions to create" - type = any - default = {} -} - -variable "fargate_profile_defaults" { - description = "Map of Fargate Profile default configurations" - type = any - default = {} + type = map(object({ + create = optional(bool) + + # Fargate profile + name = optional(string) # Will fall back to map key + subnet_ids = optional(list(string)) + selectors = optional(list(object({ + labels = optional(map(string)) + namespace = string + }))) + timeouts = optional(object({ + create = optional(string) + delete = optional(string) + })) + + # IAM role + create_iam_role = optional(bool) + iam_role_arn = optional(string) + iam_role_name = optional(string) + iam_role_use_name_prefix = optional(bool) + iam_role_path = optional(string) + iam_role_description = optional(string) + iam_role_permissions_boundary = optional(string) + iam_role_tags = optional(map(string)) + iam_role_attach_cni_policy = optional(bool) + iam_role_additional_policies = optional(map(string)) + create_iam_role_policy = optional(bool) + iam_role_policy_statements = optional(list(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + }))) + tags = optional(map(string)) + })) + default = null } ################################################################################ @@ -654,14 +781,417 @@ variable "fargate_profile_defaults" { variable "self_managed_node_groups" { description = "Map of self-managed node group definitions to create" - type = any - default = {} -} - -variable "self_managed_node_group_defaults" { - description = "Map of self-managed node group default configurations" - type = any - default = {} + type = map(object({ + create = optional(bool, true) + # Autoscaling Group + create_autoscaling_group = optional(bool) + name = optional(string) # Will fall back to map key + use_name_prefix = optional(bool) + availability_zones = optional(list(string)) + subnet_ids = optional(list(string)) + min_size = optional(number) + max_size = optional(number) + desired_size = optional(number) + desired_size_type = optional(string) + capacity_rebalance = optional(bool) + default_instance_warmup = optional(number) + protect_from_scale_in = optional(bool) + context = optional(string) + create_placement_group = optional(bool) + placement_group = optional(string) + health_check_type = optional(string) + health_check_grace_period = optional(number) + ignore_failed_scaling_activities = optional(bool) + force_delete = optional(bool) + termination_policies = optional(list(string)) + suspended_processes = optional(list(string)) + max_instance_lifetime = optional(number) + enabled_metrics = optional(list(string)) + metrics_granularity = optional(string) + initial_lifecycle_hooks = optional(list(object({ + default_result = optional(string) + heartbeat_timeout = optional(number) + lifecycle_transition = string + name = string + notification_metadata = optional(string) + notification_target_arn = optional(string) + role_arn = optional(string) + }))) + instance_maintenance_policy = optional(object({ + max_healthy_percentage = number + min_healthy_percentage = number + })) + instance_refresh = optional(object({ + preferences = optional(object({ + alarm_specification = optional(object({ + alarms = optional(list(string)) + })) + auto_rollback = optional(bool) + checkpoint_delay = optional(number) + checkpoint_percentages = optional(list(number)) + instance_warmup = optional(number) + max_healthy_percentage = optional(number) + min_healthy_percentage = optional(number) + scale_in_protected_instances = optional(string) + skip_matching = optional(bool) + standby_instances = optional(string) + })) + strategy = optional(string) + triggers = optional(list(string)) + })) + use_mixed_instances_policy = optional(bool) + mixed_instances_policy = optional(object({ + instances_distribution = optional(object({ + on_demand_allocation_strategy = optional(string) + on_demand_base_capacity = optional(number) + on_demand_percentage_above_base_capacity = optional(number) + spot_allocation_strategy = optional(string) + spot_instance_pools = optional(number) + spot_max_price = optional(string) + })) + launch_template = object({ + override = optional(list(object({ + instance_requirements = optional(object({ + accelerator_count = optional(object({ + max = optional(number) + min = optional(number) + })) + accelerator_manufacturers = optional(list(string)) + accelerator_names = optional(list(string)) + accelerator_total_memory_mib = optional(object({ + max = optional(number) + min = optional(number) + })) + accelerator_types = optional(list(string)) + allowed_instance_types = optional(list(string)) + bare_metal = optional(string) + baseline_ebs_bandwidth_mbps = optional(object({ + max = optional(number) + min = optional(number) + })) + burstable_performance = optional(string) + cpu_manufacturers = optional(list(string)) + excluded_instance_types = optional(list(string)) + instance_generations = optional(list(string)) + local_storage = optional(string) + local_storage_types = optional(list(string)) + max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number) + memory_gib_per_vcpu = optional(object({ + max = optional(number) + min = optional(number) + })) + memory_mib = optional(object({ + max = optional(number) + min = optional(number) + })) + network_bandwidth_gbps = optional(object({ + max = optional(number) + min = optional(number) + })) + network_interface_count = optional(object({ + max = optional(number) + min = optional(number) + })) + on_demand_max_price_percentage_over_lowest_price = optional(number) + require_hibernate_support = optional(bool) + spot_max_price_percentage_over_lowest_price = optional(number) + total_local_storage_gb = optional(object({ + max = optional(number) + min = optional(number) + })) + vcpu_count = optional(object({ + max = optional(number) + min = optional(number) + })) + })) + instance_type = optional(string) + launch_template_specification = optional(object({ + launch_template_id = optional(string) + launch_template_name = optional(string) + version = optional(string) + })) + weighted_capacity = optional(string) + }))) + }) + })) + timeouts = optional(object({ + delete = optional(string) + })) + autoscaling_group_tags = optional(map(string)) + # User data + ami_type = optional(string) + additional_cluster_dns_ips = optional(list(string)) + pre_bootstrap_user_data = optional(string) + post_bootstrap_user_data = optional(string) + bootstrap_extra_args = optional(string) + user_data_template_path = optional(string) + cloudinit_pre_nodeadm = optional(list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + }))) + cloudinit_post_nodeadm = optional(list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + }))) + # Launch Template + create_launch_template = optional(bool) + use_custom_launch_template = optional(bool) + launch_template_id = optional(string) + launch_template_name = optional(string) # Will fall back to map key + launch_template_use_name_prefix = optional(bool) + launch_template_version = optional(string) + launch_template_default_version = optional(string) + update_launch_template_default_version = optional(bool) + launch_template_description = optional(string) + launch_template_tags = optional(map(string)) + tag_specifications = optional(list(string)) + ebs_optimized = optional(bool) + ami_id = optional(string) + instance_type = optional(string) + key_name = optional(string) + disable_api_termination = optional(bool) + instance_initiated_shutdown_behavior = optional(string) + kernel_id = optional(string) + ram_disk_id = optional(string) + block_device_mappings = optional(map(object({ + device_name = optional(string) + ebs = optional(object({ + delete_on_termination = optional(bool) + encrypted = optional(bool) + iops = optional(number) + kms_key_id = optional(string) + snapshot_id = optional(string) + throughput = optional(number) + volume_initialization_rate = optional(number) + volume_size = optional(number) + volume_type = optional(string) + })) + no_device = optional(string) + virtual_name = optional(string) + }))) + capacity_reservation_specification = optional(object({ + capacity_reservation_preference = optional(string) + capacity_reservation_target = optional(object({ + capacity_reservation_id = optional(string) + capacity_reservation_resource_group_arn = optional(string) + })) + })) + cpu_options = optional(object({ + amd_sev_snp = optional(string) + core_count = optional(number) + threads_per_core = optional(number) + })) + credit_specification = optional(object({ + cpu_credits = optional(string) + })) + enclave_options = optional(object({ + enabled = optional(bool) + })) + instance_requirements = optional(object({ + accelerator_count = optional(object({ + max = optional(number) + min = optional(number) + })) + accelerator_manufacturers = optional(list(string)) + accelerator_names = optional(list(string)) + accelerator_total_memory_mib = optional(object({ + max = optional(number) + min = optional(number) + })) + accelerator_types = optional(list(string)) + allowed_instance_types = optional(list(string)) + bare_metal = optional(string) + baseline_ebs_bandwidth_mbps = optional(object({ + max = optional(number) + min = optional(number) + })) + burstable_performance = optional(string) + cpu_manufacturers = optional(list(string)) + excluded_instance_types = optional(list(string)) + instance_generations = optional(list(string)) + local_storage = optional(string) + local_storage_types = optional(list(string)) + max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number) + memory_gib_per_vcpu = optional(object({ + max = optional(number) + min = optional(number) + })) + memory_mib = optional(object({ + max = optional(number) + min = optional(number) + })) + network_bandwidth_gbps = optional(object({ + max = optional(number) + min = optional(number) + })) + network_interface_count = optional(object({ + max = optional(number) + min = optional(number) + })) + on_demand_max_price_percentage_over_lowest_price = optional(number) + require_hibernate_support = optional(bool) + spot_max_price_percentage_over_lowest_price = optional(number) + total_local_storage_gb = optional(object({ + max = optional(number) + min = optional(number) + })) + vcpu_count = optional(object({ + max = optional(number) + min = string + })) + })) + instance_market_options = optional(object({ + market_type = optional(string) + spot_options = optional(object({ + block_duration_minutes = optional(number) + instance_interruption_behavior = optional(string) + max_price = optional(string) + spot_instance_type = optional(string) + valid_until = optional(string) + })) + })) + license_specifications = optional(list(object({ + license_configuration_arn = string + }))) + metadata_options = optional(object({ + http_endpoint = optional(string) + http_protocol_ipv6 = optional(string) + http_put_response_hop_limit = optional(number) + http_tokens = optional(string) + instance_metadata_tags = optional(string) + })) + enable_monitoring = optional(bool) + enable_efa_support = optional(bool) + enable_efa_only = optional(bool) + efa_indices = optional(list(string)) + network_interfaces = optional(list(object({ + associate_carrier_ip_address = optional(bool) + associate_public_ip_address = optional(bool) + connection_tracking_specification = optional(object({ + tcp_established_timeout = optional(number) + udp_stream_timeout = optional(number) + udp_timeout = optional(number) + })) + delete_on_termination = optional(bool) + description = optional(string) + device_index = optional(number) + ena_srd_specification = optional(object({ + ena_srd_enabled = optional(bool) + ena_srd_udp_specification = optional(object({ + ena_srd_udp_enabled = optional(bool) + })) + })) + interface_type = optional(string) + ipv4_address_count = optional(number) + ipv4_addresses = optional(list(string)) + ipv4_prefix_count = optional(number) + ipv4_prefixes = optional(list(string)) + ipv6_address_count = optional(number) + ipv6_addresses = optional(list(string)) + ipv6_prefix_count = optional(number) + ipv6_prefixes = optional(list(string)) + network_card_index = optional(number) + network_interface_id = optional(string) + primary_ipv6 = optional(bool) + private_ip_address = optional(string) + security_groups = optional(list(string)) + subnet_id = optional(string) + }))) + placement = optional(object({ + affinity = optional(string) + availability_zone = optional(string) + group_name = optional(string) + host_id = optional(string) + host_resource_group_arn = optional(string) + partition_number = optional(number) + spread_domain = optional(string) + tenancy = optional(string) + })) + maintenance_options = optional(object({ + auto_recovery = optional(string) + })) + private_dns_name_options = optional(object({ + enable_resource_name_dns_aaaa_record = optional(bool) + enable_resource_name_dns_a_record = optional(bool) + hostname_type = optional(string) + })) + # IAM role + create_iam_instance_profile = optional(bool) + iam_instance_profile_arn = optional(string) + iam_role_name = optional(string) + iam_role_use_name_prefix = optional(bool) + iam_role_path = optional(string) + iam_role_description = optional(string) + iam_role_permissions_boundary = optional(string) + iam_role_tags = optional(map(string)) + iam_role_attach_cni_policy = optional(bool) + iam_role_additional_policies = optional(map(string)) + create_iam_role_policy = optional(bool) + iam_role_policy_statements = optional(list(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + }))) + # Access entry + create_access_entry = optional(bool) + iam_role_arn = optional(string) + # Security group + attach_cluster_primary_security_group = optional(bool, false) + create_security_group = optional(bool) + security_group_name = optional(string) + security_group_use_name_prefix = optional(bool) + security_group_description = optional(string) + security_group_ingress_rules = optional(map(object({ + name = optional(string) + cidr_ipv4 = optional(string) + cidr_ipv6 = optional(string) + description = optional(string) + from_port = optional(string) + ip_protocol = optional(string) + prefix_list_id = optional(string) + referenced_security_group_id = optional(string) + self = optional(bool) + tags = optional(map(string)) + to_port = optional(string) + }))) + security_group_egress_rules = optional(map(object({ + name = optional(string) + cidr_ipv4 = optional(string) + cidr_ipv6 = optional(string) + description = optional(string) + from_port = optional(string) + ip_protocol = optional(string) + prefix_list_id = optional(string) + referenced_security_group_id = optional(string) + self = optional(bool) + tags = optional(map(string)) + to_port = optional(string) + }))) + security_group_tags = optional(map(string)) + + tags = optional(map(string)) + })) + default = null } ################################################################################ @@ -670,14 +1200,264 @@ variable "self_managed_node_group_defaults" { variable "eks_managed_node_groups" { description = "Map of EKS managed node group definitions to create" - type = any - default = {} -} - -variable "eks_managed_node_group_defaults" { - description = "Map of EKS managed node group default configurations" - type = any - default = {} + type = map(object({ + create = optional(bool) + kubernetes_version = optional(string) + + # EKS Managed Node Group + name = optional(string) # Will fall back to map key + use_name_prefix = optional(bool) + subnet_ids = optional(list(string)) + min_size = optional(number) + max_size = optional(number) + desired_size = optional(number) + ami_id = optional(string) + ami_type = optional(string) + ami_release_version = optional(string) + use_latest_ami_release_version = optional(bool) + capacity_type = optional(string) + disk_size = optional(number) + force_update_version = optional(bool) + instance_types = optional(list(string)) + labels = optional(map(string)) + node_repair_config = optional(object({ + enabled = optional(bool) + })) + remote_access = optional(object({ + ec2_ssh_key = optional(string) + source_security_group_ids = optional(list(string)) + })) + taints = optional(map(object({ + key = string + value = optional(string) + effect = string + }))) + update_config = optional(object({ + max_unavailable = optional(number) + max_unavailable_percentage = optional(number) + })) + timeouts = optional(object({ + create = optional(string) + update = optional(string) + delete = optional(string) + })) + # User data + enable_bootstrap_user_data = optional(bool) + pre_bootstrap_user_data = optional(string) + post_bootstrap_user_data = optional(string) + bootstrap_extra_args = optional(string) + user_data_template_path = optional(string) + cloudinit_pre_nodeadm = optional(list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + }))) + cloudinit_post_nodeadm = optional(list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + }))) + # Launch Template + create_launch_template = optional(bool) + use_custom_launch_template = optional(bool) + launch_template_id = optional(string) + launch_template_name = optional(string) # Will fall back to map key + launch_template_use_name_prefix = optional(bool) + launch_template_version = optional(string) + launch_template_default_version = optional(string) + update_launch_template_default_version = optional(bool) + launch_template_description = optional(string) + launch_template_tags = optional(map(string)) + tag_specifications = optional(list(string)) + ebs_optimized = optional(bool) + key_name = optional(string) + disable_api_termination = optional(bool) + kernel_id = optional(string) + ram_disk_id = optional(string) + block_device_mappings = optional(map(object({ + device_name = optional(string) + ebs = optional(object({ + delete_on_termination = optional(bool) + encrypted = optional(bool) + iops = optional(number) + kms_key_id = optional(string) + snapshot_id = optional(string) + throughput = optional(number) + volume_initialization_rate = optional(number) + volume_size = optional(number) + volume_type = optional(string) + })) + no_device = optional(string) + virtual_name = optional(string) + }))) + capacity_reservation_specification = optional(object({ + capacity_reservation_preference = optional(string) + capacity_reservation_target = optional(object({ + capacity_reservation_id = optional(string) + capacity_reservation_resource_group_arn = optional(string) + })) + })) + cpu_options = optional(object({ + amd_sev_snp = optional(string) + core_count = optional(number) + threads_per_core = optional(number) + })) + credit_specification = optional(object({ + cpu_credits = optional(string) + })) + enclave_options = optional(object({ + enabled = optional(bool) + })) + instance_market_options = optional(object({ + market_type = optional(string) + spot_options = optional(object({ + block_duration_minutes = optional(number) + instance_interruption_behavior = optional(string) + max_price = optional(string) + spot_instance_type = optional(string) + valid_until = optional(string) + })) + })) + license_specifications = optional(list(object({ + license_configuration_arn = string + }))) + metadata_options = optional(object({ + http_endpoint = optional(string) + http_protocol_ipv6 = optional(string) + http_put_response_hop_limit = optional(number) + http_tokens = optional(string) + instance_metadata_tags = optional(string) + })) + enable_monitoring = optional(bool) + enable_efa_support = optional(bool) + enable_efa_only = optional(bool) + efa_indices = optional(list(string)) + create_placement_group = optional(bool) + placement = optional(object({ + affinity = optional(string) + availability_zone = optional(string) + group_name = optional(string) + host_id = optional(string) + host_resource_group_arn = optional(string) + partition_number = optional(number) + spread_domain = optional(string) + tenancy = optional(string) + })) + network_interfaces = optional(list(object({ + associate_carrier_ip_address = optional(bool) + associate_public_ip_address = optional(bool) + connection_tracking_specification = optional(object({ + tcp_established_timeout = optional(number) + udp_stream_timeout = optional(number) + udp_timeout = optional(number) + })) + delete_on_termination = optional(bool) + description = optional(string) + device_index = optional(number) + ena_srd_specification = optional(object({ + ena_srd_enabled = optional(bool) + ena_srd_udp_specification = optional(object({ + ena_srd_udp_enabled = optional(bool) + })) + })) + interface_type = optional(string) + ipv4_address_count = optional(number) + ipv4_addresses = optional(list(string)) + ipv4_prefix_count = optional(number) + ipv4_prefixes = optional(list(string)) + ipv6_address_count = optional(number) + ipv6_addresses = optional(list(string)) + ipv6_prefix_count = optional(number) + ipv6_prefixes = optional(list(string)) + network_card_index = optional(number) + network_interface_id = optional(string) + primary_ipv6 = optional(bool) + private_ip_address = optional(string) + security_groups = optional(list(string), []) + subnet_id = optional(string) + }))) + maintenance_options = optional(object({ + auto_recovery = optional(string) + })) + private_dns_name_options = optional(object({ + enable_resource_name_dns_aaaa_record = optional(bool) + enable_resource_name_dns_a_record = optional(bool) + hostname_type = optional(string) + })) + # IAM role + create_iam_role = optional(bool) + iam_role_arn = optional(string) + iam_role_name = optional(string) + iam_role_use_name_prefix = optional(bool) + iam_role_path = optional(string) + iam_role_description = optional(string) + iam_role_permissions_boundary = optional(string) + iam_role_tags = optional(map(string)) + iam_role_attach_cni_policy = optional(bool) + iam_role_additional_policies = optional(map(string)) + create_iam_role_policy = optional(bool) + iam_role_policy_statements = optional(list(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + }))) + # Security group + vpc_security_group_ids = optional(list(string), []) + attach_cluster_primary_security_group = optional(bool, false) + cluster_primary_security_group_id = optional(string) + create_security_group = optional(bool) + security_group_name = optional(string) + security_group_use_name_prefix = optional(bool) + security_group_description = optional(string) + security_group_ingress_rules = optional(map(object({ + name = optional(string) + cidr_ipv4 = optional(string) + cidr_ipv6 = optional(string) + description = optional(string) + from_port = optional(string) + ip_protocol = optional(string) + prefix_list_id = optional(string) + referenced_security_group_id = optional(string) + self = optional(bool) + tags = optional(map(string)) + to_port = optional(string) + }))) + security_group_egress_rules = optional(map(object({ + name = optional(string) + cidr_ipv4 = optional(string) + cidr_ipv6 = optional(string) + description = optional(string) + from_port = optional(string) + ip_protocol = optional(string) + prefix_list_id = optional(string) + referenced_security_group_id = optional(string) + self = optional(bool) + tags = optional(map(string)) + to_port = optional(string) + })), {}) + security_group_tags = optional(map(string)) + + tags = optional(map(string)) + })) + default = null } variable "putin_khuylo" { diff --git a/versions.tf b/versions.tf index 5024a4995f..71efea2e03 100644 --- a/versions.tf +++ b/versions.tf @@ -1,14 +1,14 @@ terraform { - required_version = ">= 1.3.2" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.95, < 6.0.0" + version = ">= 6.0" } tls = { source = "hashicorp/tls" - version = ">= 3.0" + version = ">= 4.0" } time = { source = "hashicorp/time" From b7eabbd3848f09e62add631e0c7683b7db0db8b9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 23 Jul 2025 20:11:33 +0000 Subject: [PATCH 887/962] chore(release): version 21.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [21.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.37.2...v21.0.0) (2025-07-23) ### ⚠ BREAKING CHANGES * Upgrade min AWS provider and Terraform versions to `6.0` and `1.5.7` respectively (#3412) ### Features * Upgrade min AWS provider and Terraform versions to `6.0` and `1.5.7` respectively ([#3412](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3412)) ([416515a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/416515a0da1ca96c539977d6460e2bc02f10b4d4)) --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb8e0eee24..f2d356db02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. +## [21.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.37.2...v21.0.0) (2025-07-23) + + +### ⚠ BREAKING CHANGES + +* Upgrade min AWS provider and Terraform versions to `6.0` and `1.5.7` respectively (#3412) + +### Features + +* Upgrade min AWS provider and Terraform versions to `6.0` and `1.5.7` respectively ([#3412](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3412)) ([416515a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/416515a0da1ca96c539977d6460e2bc02f10b4d4)) + ## [20.37.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.37.1...v20.37.2) (2025-07-17) From 97d4ebbe68a23aa431a534fd7ed56a76f9b37801 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 24 Jul 2025 09:31:24 -0500 Subject: [PATCH 888/962] =?UTF-8?q?fix:=20Correct=20logic=20to=20try=20to?= =?UTF-8?q?=20use=20module=20created=20IAM=20role=20before=20falli?= =?UTF-8?q?=E2=80=A6=20(#3433)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- examples/eks-managed-node-group/eks-al2023.tf | 5 ++--- examples/eks-managed-node-group/eks-bottlerocket.tf | 5 ++--- examples/self-managed-node-group/eks-al2023.tf | 5 ++--- examples/self-managed-node-group/eks-bottlerocket.tf | 5 ++--- main.tf | 8 ++++---- outputs.tf | 6 +++--- variables.tf | 1 + 8 files changed, 17 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 897d445231..8d3d79ad74 100644 --- a/README.md +++ b/README.md @@ -430,7 +430,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | -| [identity\_providers](#input\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA |
map(object({
client_id = string
groups_claim = optional(string)
groups_prefix = optional(string)
identity_provider_config_name = optional(string) # will fall back to map key
issuer_url = string
required_claims = optional(map(string))
username_claim = optional(string)
username_prefix = optional(string)
}))
| `null` | no | +| [identity\_providers](#input\_identity\_providers) | Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA |
map(object({
client_id = string
groups_claim = optional(string)
groups_prefix = optional(string)
identity_provider_config_name = optional(string) # will fall back to map key
issuer_url = string
required_claims = optional(map(string))
username_claim = optional(string)
username_prefix = optional(string)
tags = optional(map(string), {})
}))
| `null` | no | | [include\_oidc\_root\_ca\_thumbprint](#input\_include\_oidc\_root\_ca\_thumbprint) | Determines whether to include the root CA thumbprint in the OpenID Connect (OIDC) identity provider's server certificate(s) | `bool` | `true` | no | | [ip\_family](#input\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created | `string` | `"ipv4"` | no | | [kms\_key\_administrators](#input\_kms\_key\_administrators) | A list of IAM ARNs for [key administrators](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-administrators). If no value is provided, the current caller identity is used to ensure at least one key admin is available | `list(string)` | `[]` | no | diff --git a/examples/eks-managed-node-group/eks-al2023.tf b/examples/eks-managed-node-group/eks-al2023.tf index 7203c01c81..69897a5a04 100644 --- a/examples/eks-managed-node-group/eks-al2023.tf +++ b/examples/eks-managed-node-group/eks-al2023.tf @@ -1,7 +1,6 @@ module "eks_al2023" { - # source = "terraform-aws-modules/eks/aws" - # version = "~> 20.0" - source = "../.." + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" name = "${local.name}-al2023" kubernetes_version = "1.33" diff --git a/examples/eks-managed-node-group/eks-bottlerocket.tf b/examples/eks-managed-node-group/eks-bottlerocket.tf index 2fd3816dbe..a4a9928d0f 100644 --- a/examples/eks-managed-node-group/eks-bottlerocket.tf +++ b/examples/eks-managed-node-group/eks-bottlerocket.tf @@ -1,7 +1,6 @@ module "eks_bottlerocket" { - # source = "terraform-aws-modules/eks/aws" - # version = "~> 20.0" - source = "../.." + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" name = "${local.name}-bottlerocket" kubernetes_version = "1.33" diff --git a/examples/self-managed-node-group/eks-al2023.tf b/examples/self-managed-node-group/eks-al2023.tf index e074735ebb..ca17ac67d9 100644 --- a/examples/self-managed-node-group/eks-al2023.tf +++ b/examples/self-managed-node-group/eks-al2023.tf @@ -1,7 +1,6 @@ module "eks_al2023" { - # source = "terraform-aws-modules/eks/aws" - # version = "~> 20.0" - source = "../.." + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" name = "${local.name}-al2023" kubernetes_version = "1.33" diff --git a/examples/self-managed-node-group/eks-bottlerocket.tf b/examples/self-managed-node-group/eks-bottlerocket.tf index 6f0a2dc5ef..be9b2450a9 100644 --- a/examples/self-managed-node-group/eks-bottlerocket.tf +++ b/examples/self-managed-node-group/eks-bottlerocket.tf @@ -1,7 +1,6 @@ module "eks_bottlerocket" { - # source = "terraform-aws-modules/eks/aws" - # version = "~> 20.0" - source = "../.." + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" name = "${local.name}-bottlerocket" kubernetes_version = "1.33" diff --git a/main.tf b/main.tf index 94989384e2..b8811e0997 100644 --- a/main.tf +++ b/main.tf @@ -62,7 +62,7 @@ resource "aws_eks_cluster" "this" { content { enabled = compute_config.value.enabled node_pools = compute_config.value.node_pools - node_role_arn = compute_config.value.node_pools != null ? try(compute_config.value.node_role_arn, aws_iam_role.eks_auto[0].arn, null) : null + node_role_arn = compute_config.value.node_pools != null ? try(aws_iam_role.eks_auto[0].arn, compute_config.value.node_role_arn) : null } } @@ -444,7 +444,7 @@ data "tls_certificate" "this" { # Not available on outposts count = local.create_oidc_provider && var.include_oidc_root_ca_thumbprint ? 1 : 0 - url = local.dualstack_oidc_issuer_url + url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer } resource "aws_iam_openid_connect_provider" "oidc_provider" { @@ -453,7 +453,7 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { client_id_list = distinct(compact(concat(["sts.amazonaws.com"], var.openid_connect_audiences))) thumbprint_list = concat(local.oidc_root_ca_thumbprint, var.custom_oidc_thumbprints) - url = local.dualstack_oidc_issuer_url + url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer tags = merge( { Name = "${var.name}-eks-irsa" }, @@ -856,7 +856,7 @@ resource "aws_eks_identity_provider_config" "this" { client_id = each.value.client_id groups_claim = each.value.groups_claim groups_prefix = each.value.groups_prefix - identity_provider_config_name = try(each.value.identity_provider_config_name, each.key) + identity_provider_config_name = coalesce(each.value.identity_provider_config_name, each.key) issuer_url = each.value.issuer_url required_claims = each.value.required_claims username_claim = each.value.username_claim diff --git a/outputs.tf b/outputs.tf index 95afd4365b..4663b83639 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,6 +1,5 @@ locals { - # https://github.com/aws/containers-roadmap/issues/2038#issuecomment-2278450601 - dualstack_oidc_issuer_url = try(replace(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://oidc.eks.", "https://oidc-eks."), ".amazonaws.com/", ".api.aws/"), null) + } ################################################################################ @@ -59,7 +58,8 @@ output "cluster_oidc_issuer_url" { output "cluster_dualstack_oidc_issuer_url" { description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" - value = local.dualstack_oidc_issuer_url + # https://github.com/aws/containers-roadmap/issues/2038#issuecomment-2278450601 + value = try(replace(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://oidc.eks.", "https://oidc-eks."), ".amazonaws.com/", ".api.aws/"), null) } output "cluster_version" { diff --git a/variables.tf b/variables.tf index 6b3e972a62..d8ade28a4c 100644 --- a/variables.tf +++ b/variables.tf @@ -660,6 +660,7 @@ variable "identity_providers" { required_claims = optional(map(string)) username_claim = optional(string) username_prefix = optional(string) + tags = optional(map(string), {}) })) default = null } From 0049a9b9d0d9626f564097cebb301ca16bc7b867 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 24 Jul 2025 14:31:53 +0000 Subject: [PATCH 889/962] chore(release): version 21.0.1 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [21.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.0...v21.0.1) (2025-07-24) ### Bug Fixes * Correct logic to try to use module created IAM role before falli… ([#3433](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3433)) ([97d4ebb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/97d4ebbe68a23aa431a534fd7ed56a76f9b37801)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2d356db02..ef9a031867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.0...v21.0.1) (2025-07-24) + + +### Bug Fixes + +* Correct logic to try to use module created IAM role before falli… ([#3433](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3433)) ([97d4ebb](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/97d4ebbe68a23aa431a534fd7ed56a76f9b37801)) + ## [21.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v20.37.2...v21.0.0) (2025-07-23) From b37368fdbc608a026f9c17952d964467f5e44e8a Mon Sep 17 00:00:00 2001 From: Brandon Kneeld Date: Thu, 24 Jul 2025 14:58:41 -0400 Subject: [PATCH 890/962] fix: Move `encryption_config` default for `resources` out of type definition and to default variable value to allow disabling encryption (#3436) * Update variables.tf Fixes inability to provide default {} encryption_config * Readme update * fix syntax * syntax * fix: Run pre-commit --------- Co-authored-by: Bryant Biggs --- README.md | 2 +- variables.tf | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8d3d79ad74..7ebc0f4555 100644 --- a/README.md +++ b/README.md @@ -411,7 +411,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | | [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled | `bool` | `true` | no | | [enabled\_log\_types](#input\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | -| [encryption\_config](#input\_encryption\_config) | Configuration block with encryption configuration for the cluster |
object({
provider_key_arn = optional(string)
resources = optional(list(string), ["secrets"])
})
| `{}` | no | +| [encryption\_config](#input\_encryption\_config) | Configuration block with encryption configuration for the cluster |
object({
provider_key_arn = optional(string)
resources = optional(list(string))
})
|
{
"resources": [
"secrets"
]
}
| no | | [encryption\_policy\_description](#input\_encryption\_policy\_description) | Description of the cluster encryption policy created | `string` | `"Cluster encryption policy to allow cluster role to utilize CMK provided"` | no | | [encryption\_policy\_name](#input\_encryption\_policy\_name) | Name to use on cluster encryption policy created | `string` | `null` | no | | [encryption\_policy\_path](#input\_encryption\_policy\_path) | Cluster encryption policy path | `string` | `null` | no | diff --git a/variables.tf b/variables.tf index d8ade28a4c..81211a8eb5 100644 --- a/variables.tf +++ b/variables.tf @@ -165,9 +165,11 @@ variable "encryption_config" { description = "Configuration block with encryption configuration for the cluster" type = object({ provider_key_arn = optional(string) - resources = optional(list(string), ["secrets"]) + resources = optional(list(string)) }) - default = {} + default = { + resources = ["secrets"] + } } variable "attach_encryption_policy" { From 7efa43f11c193df8da3d5687b292b14517288f59 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 24 Jul 2025 18:59:07 +0000 Subject: [PATCH 891/962] chore(release): version 21.0.2 [skip ci] ## [21.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.1...v21.0.2) (2025-07-24) ### Bug Fixes * Move `encryption_config` default for `resources` out of type definition and to default variable value to allow disabling encryption ([#3436](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3436)) ([b37368f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b37368fdbc608a026f9c17952d964467f5e44e8a)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef9a031867..080aabfe3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.1...v21.0.2) (2025-07-24) + + +### Bug Fixes + +* Move `encryption_config` default for `resources` out of type definition and to default variable value to allow disabling encryption ([#3436](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3436)) ([b37368f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b37368fdbc608a026f9c17952d964467f5e44e8a)) + ## [21.0.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.0...v21.0.1) (2025-07-24) From 8807e0bb55fdc49ed894b5b51c14131526dbfb91 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 24 Jul 2025 15:28:35 -0500 Subject: [PATCH 892/962] fix: Correct variable defaults for `ami_id` and `kubernetes_version` (#3437) --- README.md | 2 +- modules/eks-managed-node-group/variables.tf | 1 + modules/self-managed-node-group/variables.tf | 1 + node_groups.tf | 137 ++++++++++--------- variables.tf | 5 +- 5 files changed, 76 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 7ebc0f4555..83365b1d0f 100644 --- a/README.md +++ b/README.md @@ -471,7 +471,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [security\_group\_name](#input\_security\_group\_name) | Name to use on cluster security group created | `string` | `null` | no | | [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the cluster security group created | `map(string)` | `{}` | no | | [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix | `bool` | `true` | no | -| [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create |
map(object({
create = optional(bool, true)
# Autoscaling Group
create_autoscaling_group = optional(bool)
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
availability_zones = optional(list(string))
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
desired_size_type = optional(string)
capacity_rebalance = optional(bool)
default_instance_warmup = optional(number)
protect_from_scale_in = optional(bool)
context = optional(string)
create_placement_group = optional(bool)
placement_group = optional(string)
health_check_type = optional(string)
health_check_grace_period = optional(number)
ignore_failed_scaling_activities = optional(bool)
force_delete = optional(bool)
termination_policies = optional(list(string))
suspended_processes = optional(list(string))
max_instance_lifetime = optional(number)
enabled_metrics = optional(list(string))
metrics_granularity = optional(string)
initial_lifecycle_hooks = optional(list(object({
default_result = optional(string)
heartbeat_timeout = optional(number)
lifecycle_transition = string
name = string
notification_metadata = optional(string)
notification_target_arn = optional(string)
role_arn = optional(string)
})))
instance_maintenance_policy = optional(object({
max_healthy_percentage = number
min_healthy_percentage = number
}))
instance_refresh = optional(object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string)
triggers = optional(list(string))
}))
use_mixed_instances_policy = optional(bool)
mixed_instances_policy = optional(object({
instances_distribution = optional(object({
on_demand_allocation_strategy = optional(string)
on_demand_base_capacity = optional(number)
on_demand_percentage_above_base_capacity = optional(number)
spot_allocation_strategy = optional(string)
spot_instance_pools = optional(number)
spot_max_price = optional(string)
}))
launch_template = object({
override = optional(list(object({
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = optional(number)
}))
}))
instance_type = optional(string)
launch_template_specification = optional(object({
launch_template_id = optional(string)
launch_template_name = optional(string)
version = optional(string)
}))
weighted_capacity = optional(string)
})))
})
}))
timeouts = optional(object({
delete = optional(string)
}))
autoscaling_group_tags = optional(map(string))
# User data
ami_type = optional(string)
additional_cluster_dns_ips = optional(list(string))
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
ami_id = optional(string)
instance_type = optional(string)
key_name = optional(string)
disable_api_termination = optional(bool)
instance_initiated_shutdown_behavior = optional(string)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = string
}))
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string))
subnet_id = optional(string)
})))
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_instance_profile = optional(bool)
iam_instance_profile_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Access entry
create_access_entry = optional(bool)
iam_role_arn = optional(string)
# Security group
attach_cluster_primary_security_group = optional(bool, false)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | +| [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# Autoscaling Group
create_autoscaling_group = optional(bool)
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
availability_zones = optional(list(string))
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
desired_size_type = optional(string)
capacity_rebalance = optional(bool)
default_instance_warmup = optional(number)
protect_from_scale_in = optional(bool)
context = optional(string)
create_placement_group = optional(bool)
placement_group = optional(string)
health_check_type = optional(string)
health_check_grace_period = optional(number)
ignore_failed_scaling_activities = optional(bool)
force_delete = optional(bool)
termination_policies = optional(list(string))
suspended_processes = optional(list(string))
max_instance_lifetime = optional(number)
enabled_metrics = optional(list(string))
metrics_granularity = optional(string)
initial_lifecycle_hooks = optional(list(object({
default_result = optional(string)
heartbeat_timeout = optional(number)
lifecycle_transition = string
name = string
notification_metadata = optional(string)
notification_target_arn = optional(string)
role_arn = optional(string)
})))
instance_maintenance_policy = optional(object({
max_healthy_percentage = number
min_healthy_percentage = number
}))
instance_refresh = optional(object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string)
triggers = optional(list(string))
}))
use_mixed_instances_policy = optional(bool)
mixed_instances_policy = optional(object({
instances_distribution = optional(object({
on_demand_allocation_strategy = optional(string)
on_demand_base_capacity = optional(number)
on_demand_percentage_above_base_capacity = optional(number)
spot_allocation_strategy = optional(string)
spot_instance_pools = optional(number)
spot_max_price = optional(string)
}))
launch_template = object({
override = optional(list(object({
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = optional(number)
}))
}))
instance_type = optional(string)
launch_template_specification = optional(object({
launch_template_id = optional(string)
launch_template_name = optional(string)
version = optional(string)
}))
weighted_capacity = optional(string)
})))
})
}))
timeouts = optional(object({
delete = optional(string)
}))
autoscaling_group_tags = optional(map(string))
# User data
ami_type = optional(string)
additional_cluster_dns_ips = optional(list(string))
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
ami_id = optional(string)
instance_type = optional(string)
key_name = optional(string)
disable_api_termination = optional(bool)
instance_initiated_shutdown_behavior = optional(string)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = string
}))
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string))
subnet_id = optional(string)
})))
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_instance_profile = optional(bool)
iam_instance_profile_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Access entry
create_access_entry = optional(bool)
iam_role_arn = optional(string)
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | | [service\_ipv4\_cidr](#input\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [service\_ipv6\_cidr](#input\_service\_ipv6\_cidr) | The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster | `string` | `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets | `list(string)` | `[]` | no | diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 281a32f63b..6e65933e51 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -163,6 +163,7 @@ variable "ami_id" { description = "The AMI from which to launch the instance. If not supplied, EKS will use its own default image" type = string default = "" + nullable = false } variable "key_name" { diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index a07d429f6b..e3ef63e053 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -359,6 +359,7 @@ variable "ami_id" { description = "The AMI from which to launch the instance" type = string default = "" + nullable = false } variable "ami_type" { diff --git a/node_groups.tf b/node_groups.tf index 501da7e0ff..5a1655613e 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -274,7 +274,8 @@ module "eks_managed_node_group" { account_id = local.account_id cluster_name = time_sleep.this[0].triggers["name"] - kubernetes_version = try(each.value.kubernetes_version, time_sleep.this[0].triggers["kubernetes_version"]) + kubernetes_version = each.value.kubernetes_version != null ? each.value.kubernetes_version : time_sleep.this[0].triggers["kubernetes_version"] + # EKS Managed Node Group name = coalesce(each.value.name, each.key) @@ -445,88 +446,88 @@ module "self_managed_node_group" { autoscaling_group_tags = each.value.autoscaling_group_tags # User data - ami_type = try(each.value.ami_type, null) + ami_type = each.value.ami_type cluster_endpoint = try(time_sleep.this[0].triggers["endpoint"], "") cluster_auth_base64 = try(time_sleep.this[0].triggers["certificate_authority_data"], "") cluster_service_cidr = try(time_sleep.this[0].triggers["service_cidr"], "") - additional_cluster_dns_ips = try(each.value.additional_cluster_dns_ips, null) + additional_cluster_dns_ips = each.value.additional_cluster_dns_ips cluster_ip_family = var.ip_family - pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, null) - post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, null) - bootstrap_extra_args = try(each.value.bootstrap_extra_args, null) - user_data_template_path = try(each.value.user_data_template_path, null) - cloudinit_pre_nodeadm = try(each.value.cloudinit_pre_nodeadm, null) - cloudinit_post_nodeadm = try(each.value.cloudinit_post_nodeadm, null) + pre_bootstrap_user_data = each.value.pre_bootstrap_user_data + post_bootstrap_user_data = each.value.post_bootstrap_user_data + bootstrap_extra_args = each.value.bootstrap_extra_args + user_data_template_path = each.value.user_data_template_path + cloudinit_pre_nodeadm = each.value.cloudinit_pre_nodeadm + cloudinit_post_nodeadm = each.value.cloudinit_post_nodeadm # Launch Template - create_launch_template = try(each.value.create_launch_template, null) - launch_template_id = try(each.value.launch_template_id, null) + create_launch_template = each.value.create_launch_template + launch_template_id = each.value.launch_template_id launch_template_name = coalesce(each.value.launch_template_name, each.key) - launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, null) - launch_template_version = try(each.value.launch_template_version, null) - launch_template_default_version = try(each.value.launch_template_default_version, null) - update_launch_template_default_version = try(each.value.update_launch_template_default_version, null) + launch_template_use_name_prefix = each.value.launch_template_use_name_prefix + launch_template_version = each.value.launch_template_version + launch_template_default_version = each.value.launch_template_default_version + update_launch_template_default_version = each.value.update_launch_template_default_version launch_template_description = coalesce(each.value.launch_template_description, "Custom launch template for ${coalesce(each.value.name, each.key)} self managed node group") - launch_template_tags = try(each.value.launch_template_tags, null) - tag_specifications = try(each.value.tag_specifications, null) - - ebs_optimized = try(each.value.ebs_optimized, null) - ami_id = try(each.value.ami_id, null) - kubernetes_version = try(each.value.kubernetes_version, time_sleep.this[0].triggers["kubernetes_version"]) - instance_type = try(each.value.instance_type, null) - key_name = try(each.value.key_name, null) - - disable_api_termination = try(each.value.disable_api_termination, null) - instance_initiated_shutdown_behavior = try(each.value.instance_initiated_shutdown_behavior, null) - kernel_id = try(each.value.kernel_id, null) - ram_disk_id = try(each.value.ram_disk_id, null) - - block_device_mappings = try(each.value.block_device_mappings, null) - capacity_reservation_specification = try(each.value.capacity_reservation_specification, null) - cpu_options = try(each.value.cpu_options, null) - credit_specification = try(each.value.credit_specification, null) - enclave_options = try(each.value.enclave_options, null) - instance_requirements = try(each.value.instance_requirements, null) - instance_market_options = try(each.value.instance_market_options, null) - license_specifications = try(each.value.license_specifications, null) - metadata_options = try(each.value.metadata_options, null) - enable_monitoring = try(each.value.enable_monitoring, null) - enable_efa_support = try(each.value.enable_efa_support, null) - enable_efa_only = try(each.value.enable_efa_only, null) - efa_indices = try(each.value.efa_indices, null) - network_interfaces = try(each.value.network_interfaces, null) - placement = try(each.value.placement, null) - maintenance_options = try(each.value.maintenance_options, null) - private_dns_name_options = try(each.value.private_dns_name_options, null) + launch_template_tags = each.value.launch_template_tags + tag_specifications = each.value.tag_specifications + + ebs_optimized = each.value.ebs_optimized + ami_id = each.value.ami_id + kubernetes_version = each.value.kubernetes_version != null ? each.value.kubernetes_version : time_sleep.this[0].triggers["kubernetes_version"] + instance_type = each.value.instance_type + key_name = each.value.key_name + + disable_api_termination = each.value.disable_api_termination + instance_initiated_shutdown_behavior = each.value.instance_initiated_shutdown_behavior + kernel_id = each.value.kernel_id + ram_disk_id = each.value.ram_disk_id + + block_device_mappings = each.value.block_device_mappings + capacity_reservation_specification = each.value.capacity_reservation_specification + cpu_options = each.value.cpu_options + credit_specification = each.value.credit_specification + enclave_options = each.value.enclave_options + instance_requirements = each.value.instance_requirements + instance_market_options = each.value.instance_market_options + license_specifications = each.value.license_specifications + metadata_options = each.value.metadata_options + enable_monitoring = each.value.enable_monitoring + enable_efa_support = each.value.enable_efa_support + enable_efa_only = each.value.enable_efa_only + efa_indices = each.value.efa_indices + network_interfaces = each.value.network_interfaces + placement = each.value.placement + maintenance_options = each.value.maintenance_options + private_dns_name_options = each.value.private_dns_name_options # IAM role - create_iam_instance_profile = try(each.value.create_iam_instance_profile, null) - iam_instance_profile_arn = try(each.value.iam_instance_profile_arn, null) - iam_role_name = try(each.value.iam_role_name, null) - iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, true) - iam_role_path = try(each.value.iam_role_path, null) - iam_role_description = try(each.value.iam_role_description, null) - iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, null) - iam_role_tags = try(each.value.iam_role_tags, null) - iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, null) + create_iam_instance_profile = each.value.create_iam_instance_profile + iam_instance_profile_arn = each.value.iam_instance_profile_arn + iam_role_name = each.value.iam_role_name + iam_role_use_name_prefix = each.value.iam_role_use_name_prefix + iam_role_path = each.value.iam_role_path + iam_role_description = each.value.iam_role_description + iam_role_permissions_boundary = each.value.iam_role_permissions_boundary + iam_role_tags = each.value.iam_role_tags + iam_role_attach_cni_policy = each.value.iam_role_attach_cni_policy iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", null) - create_iam_role_policy = try(each.value.create_iam_role_policy, null) - iam_role_policy_statements = try(each.value.iam_role_policy_statements, null) + create_iam_role_policy = each.value.create_iam_role_policy + iam_role_policy_statements = each.value.iam_role_policy_statements # Access entry - create_access_entry = try(each.value.create_access_entry, null) - iam_role_arn = try(each.value.iam_role_arn, null) + create_access_entry = each.value.create_access_entry + iam_role_arn = each.value.iam_role_arn # Security group - vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, []))) - cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null - create_security_group = try(each.value.create_security_group, null) - security_group_name = try(each.value.security_group_name, null) - security_group_use_name_prefix = try(each.value.security_group_use_name_prefix, null) - security_group_description = try(each.value.security_group_description, null) - security_group_ingress_rules = try(each.value.security_group_ingress_rules, null) - security_group_egress_rules = try(each.value.security_group_egress_rules, null) - security_group_tags = try(each.value.security_group_tags, null) + vpc_security_group_ids = compact(concat([local.node_security_group_id], each.value.vpc_security_group_ids)) + cluster_primary_security_group_id = each.value.attach_cluster_primary_security_group ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null + create_security_group = each.value.create_security_group + security_group_name = each.value.security_group_name + security_group_use_name_prefix = each.value.security_group_use_name_prefix + security_group_description = each.value.security_group_description + security_group_ingress_rules = each.value.security_group_ingress_rules + security_group_egress_rules = each.value.security_group_egress_rules + security_group_tags = each.value.security_group_tags tags = merge( var.tags, diff --git a/variables.tf b/variables.tf index 81211a8eb5..fa7c0604f5 100644 --- a/variables.tf +++ b/variables.tf @@ -785,7 +785,9 @@ variable "fargate_profiles" { variable "self_managed_node_groups" { description = "Map of self-managed node group definitions to create" type = map(object({ - create = optional(bool, true) + create = optional(bool) + kubernetes_version = optional(string) + # Autoscaling Group create_autoscaling_group = optional(bool) name = optional(string) # Will fall back to map key @@ -1159,6 +1161,7 @@ variable "self_managed_node_groups" { create_access_entry = optional(bool) iam_role_arn = optional(string) # Security group + vpc_security_group_ids = optional(list(string), []) attach_cluster_primary_security_group = optional(bool, false) create_security_group = optional(bool) security_group_name = optional(string) From 41d31db4e2a4e2a794eb0b933898511761c5c697 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 24 Jul 2025 20:28:57 +0000 Subject: [PATCH 893/962] chore(release): version 21.0.3 [skip ci] ## [21.0.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.2...v21.0.3) (2025-07-24) ### Bug Fixes * Correct variable defaults for `ami_id` and `kubernetes_version` ([#3437](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3437)) ([8807e0b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8807e0bb55fdc49ed894b5b51c14131526dbfb91)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 080aabfe3c..a0ac6942dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.0.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.2...v21.0.3) (2025-07-24) + + +### Bug Fixes + +* Correct variable defaults for `ami_id` and `kubernetes_version` ([#3437](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3437)) ([8807e0b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8807e0bb55fdc49ed894b5b51c14131526dbfb91)) + ## [21.0.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.1...v21.0.2) (2025-07-24) From 6b8a3d94777346d79a64ccd8287c96b525348013 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 25 Jul 2025 07:19:45 -0500 Subject: [PATCH 894/962] fix: Correct encryption configuration enable logic; avoid creating Auto Mode policy when Auto Mode is not enabled (#3439) --- README.md | 2 +- main.tf | 8 ++++---- variables.tf | 6 ++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 83365b1d0f..fec86afc14 100644 --- a/README.md +++ b/README.md @@ -411,7 +411,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | | [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled | `bool` | `true` | no | | [enabled\_log\_types](#input\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` |
[
"audit",
"api",
"authenticator"
]
| no | -| [encryption\_config](#input\_encryption\_config) | Configuration block with encryption configuration for the cluster |
object({
provider_key_arn = optional(string)
resources = optional(list(string))
})
|
{
"resources": [
"secrets"
]
}
| no | +| [encryption\_config](#input\_encryption\_config) | Configuration block with encryption configuration for the cluster |
object({
provider_key_arn = optional(string)
resources = optional(list(string), ["secrets"])
})
| `{}` | no | | [encryption\_policy\_description](#input\_encryption\_policy\_description) | Description of the cluster encryption policy created | `string` | `"Cluster encryption policy to allow cluster role to utilize CMK provided"` | no | | [encryption\_policy\_name](#input\_encryption\_policy\_name) | Name to use on cluster encryption policy created | `string` | `null` | no | | [encryption\_policy\_path](#input\_encryption\_policy\_path) | Cluster encryption policy path | `string` | `null` | no | diff --git a/main.tf b/main.tf index b8811e0997..4e423488b5 100644 --- a/main.tf +++ b/main.tf @@ -24,7 +24,7 @@ locals { role_arn = try(aws_iam_role.this[0].arn, var.iam_role_arn) create_outposts_local_cluster = var.outpost_config != null - enable_encryption_config = length(var.encryption_config) > 0 && !local.create_outposts_local_cluster + enable_encryption_config = var.encryption_config != null && !local.create_outposts_local_cluster auto_mode_enabled = try(var.compute_config.enabled, false) } @@ -590,7 +590,7 @@ resource "aws_iam_policy" "cluster_encryption" { } data "aws_iam_policy_document" "custom" { - count = local.create_iam_role && var.enable_auto_mode_custom_tags ? 1 : 0 + count = local.create_iam_role && local.auto_mode_enabled && var.enable_auto_mode_custom_tags ? 1 : 0 dynamic "statement" { for_each = var.enable_auto_mode_custom_tags ? [1] : [] @@ -724,7 +724,7 @@ data "aws_iam_policy_document" "custom" { } resource "aws_iam_policy" "custom" { - count = local.create_iam_role && var.enable_auto_mode_custom_tags ? 1 : 0 + count = local.create_iam_role && local.auto_mode_enabled && var.enable_auto_mode_custom_tags ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null @@ -737,7 +737,7 @@ resource "aws_iam_policy" "custom" { } resource "aws_iam_role_policy_attachment" "custom" { - count = local.create_iam_role && var.enable_auto_mode_custom_tags ? 1 : 0 + count = local.create_iam_role && local.auto_mode_enabled && var.enable_auto_mode_custom_tags ? 1 : 0 policy_arn = aws_iam_policy.custom[0].arn role = aws_iam_role.this[0].name diff --git a/variables.tf b/variables.tf index fa7c0604f5..5e23df6c98 100644 --- a/variables.tf +++ b/variables.tf @@ -165,11 +165,9 @@ variable "encryption_config" { description = "Configuration block with encryption configuration for the cluster" type = object({ provider_key_arn = optional(string) - resources = optional(list(string)) + resources = optional(list(string), ["secrets"]) }) - default = { - resources = ["secrets"] - } + default = {} } variable "attach_encryption_policy" { From 074abf14097264dd13fe3a521cd1d0fe76f6b4ef Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 25 Jul 2025 12:20:09 +0000 Subject: [PATCH 895/962] chore(release): version 21.0.4 [skip ci] ## [21.0.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.3...v21.0.4) (2025-07-25) ### Bug Fixes * Correct encryption configuration enable logic; avoid creating Auto Mode policy when Auto Mode is not enabled ([#3439](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3439)) ([6b8a3d9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6b8a3d94777346d79a64ccd8287c96b525348013)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0ac6942dc..09ade1ffb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.0.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.3...v21.0.4) (2025-07-25) + + +### Bug Fixes + +* Correct encryption configuration enable logic; avoid creating Auto Mode policy when Auto Mode is not enabled ([#3439](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3439)) ([6b8a3d9](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6b8a3d94777346d79a64ccd8287c96b525348013)) + ## [21.0.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.2...v21.0.3) (2025-07-24) From 55d7fa23a356f518ae7b73ec2ddb0ab5947f9a42 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 29 Jul 2025 10:40:12 -0500 Subject: [PATCH 896/962] fix: Correct addon logic lookup to pull latest addon version (#3449) --- main.tf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/main.tf b/main.tf index 4e423488b5..48207dc238 100644 --- a/main.tf +++ b/main.tf @@ -766,7 +766,7 @@ resource "aws_eks_addon" "this" { cluster_name = aws_eks_cluster.this[0].id addon_name = coalesce(each.value.name, each.key) - addon_version = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) + addon_version = coalesce(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) configuration_values = each.value.configuration_values dynamic "pod_identity_association" { @@ -784,9 +784,9 @@ resource "aws_eks_addon" "this" { service_account_role_arn = each.value.service_account_role_arn timeouts { - create = try(each.value.timeouts.create, var.addons_timeouts.create, null) - update = try(each.value.timeouts.update, var.addons_timeouts.update, null) - delete = try(each.value.timeouts.delete, var.addons_timeouts.delete, null) + create = try(coalesce(each.value.timeouts.create, var.addons_timeouts.create), null) + update = try(coalesce(each.value.timeouts.update, var.addons_timeouts.update), null) + delete = try(coalesce(each.value.timeouts.delete, var.addons_timeouts.delete), null) } tags = merge( @@ -811,7 +811,7 @@ resource "aws_eks_addon" "before_compute" { cluster_name = aws_eks_cluster.this[0].id addon_name = coalesce(each.value.name, each.key) - addon_version = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) + addon_version = coalesce(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) configuration_values = each.value.configuration_values dynamic "pod_identity_association" { @@ -829,9 +829,9 @@ resource "aws_eks_addon" "before_compute" { service_account_role_arn = each.value.service_account_role_arn timeouts { - create = try(each.value.timeouts.create, var.addons_timeouts.create, null) - update = try(each.value.timeouts.update, var.addons_timeouts.update, null) - delete = try(each.value.timeouts.delete, var.addons_timeouts.delete, null) + create = try(coalesce(each.value.timeouts.create, var.addons_timeouts.create), null) + update = try(coalesce(each.value.timeouts.update, var.addons_timeouts.update), null) + delete = try(coalesce(each.value.timeouts.delete, var.addons_timeouts.delete), null) } tags = merge( From 1f2f95af92c7ef4d4e26330ab2cb27a7035491b6 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 29 Jul 2025 15:40:39 +0000 Subject: [PATCH 897/962] chore(release): version 21.0.5 [skip ci] ## [21.0.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.4...v21.0.5) (2025-07-29) ### Bug Fixes * Correct addon logic lookup to pull latest addon version ([#3449](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3449)) ([55d7fa2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/55d7fa23a356f518ae7b73ec2ddb0ab5947f9a42)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09ade1ffb4..be461948e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.0.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.4...v21.0.5) (2025-07-29) + + +### Bug Fixes + +* Correct addon logic lookup to pull latest addon version ([#3449](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3449)) ([55d7fa2](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/55d7fa23a356f518ae7b73ec2ddb0ab5947f9a42)) + ## [21.0.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.3...v21.0.4) (2025-07-25) From 5322bf72fbbff4afb6a02ae283b21419d9de5b17 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 30 Jul 2025 12:04:27 -0500 Subject: [PATCH 898/962] fix: Allow `instance_requirements` to be set in self-managed node groups (#3455) --- modules/self-managed-node-group/main.tf | 2 +- tests/self-managed-node-group/main.tf | 141 ++++++++++++------------ 2 files changed, 74 insertions(+), 69 deletions(-) diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 710d276602..b30476cbdf 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -321,7 +321,7 @@ resource "aws_launch_template" "this" { } } - instance_type = var.instance_type + instance_type = var.instance_requirements != null ? null : var.instance_type kernel_id = var.kernel_id key_name = var.key_name diff --git a/tests/self-managed-node-group/main.tf b/tests/self-managed-node-group/main.tf index 8629039e7d..9fc3fdc6b1 100644 --- a/tests/self-managed-node-group/main.tf +++ b/tests/self-managed-node-group/main.tf @@ -164,7 +164,79 @@ module "eks" { } } - # Complete + instance_attributes = { + name = "instance-attributes" + + min_size = 1 + max_size = 2 + desired_size = 1 + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + EOT + content_type = "application/node.eks.aws" + }] + + instance_type = null + + # launch template configuration + instance_requirements = { + cpu_manufacturers = ["intel"] + instance_generations = ["current", "previous"] + spot_max_price_percentage_over_lowest_price = 100 + + memory_mib = { + min = 8192 + } + + vcpu_count = { + min = 1 + } + + allowed_instance_types = ["t*", "m*"] + } + + use_mixed_instances_policy = true + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 0 + on_demand_allocation_strategy = "lowest-price" + spot_allocation_strategy = "price-capacity-optimized" + } + + # ASG configuration + launch_template = { + override = [ + { + instance_requirements = { + cpu_manufacturers = ["intel"] + instance_generations = ["current", "previous"] + spot_max_price_percentage_over_lowest_price = 100 + + memory_mib = { + min = 8192 + } + + vcpu_count = { + min = 1 + } + + allowed_instance_types = ["t*", "m*"] + } + } + ] + } + } + } + complete = { name = "complete-self-mng" use_name_prefix = false @@ -212,73 +284,6 @@ module "eks" { } } - instance_attributes = { - name = "instance-attributes" - - min_size = 1 - max_size = 2 - desired_size = 1 - - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'" - - cloudinit_pre_nodeadm = [{ - content = <<-EOT - --- - apiVersion: node.eks.aws/v1alpha1 - kind: NodeConfig - spec: - kubelet: - config: - shutdownGracePeriod: 30s - EOT - content_type = "application/node.eks.aws" - }] - - instance_type = null - - # launch template configuration - instance_requirements = { - cpu_manufacturers = ["intel"] - instance_generations = ["current", "previous"] - spot_max_price_percentage_over_lowest_price = 100 - - vcpu_count = { - min = 1 - } - - allowed_instance_types = ["t*", "m*"] - } - - use_mixed_instances_policy = true - mixed_instances_policy = { - instances_distribution = { - on_demand_base_capacity = 0 - on_demand_percentage_above_base_capacity = 0 - on_demand_allocation_strategy = "lowest-price" - spot_allocation_strategy = "price-capacity-optimized" - } - - # ASG configuration - launch_template = { - override = [ - { - instance_requirements = { - cpu_manufacturers = ["intel"] - instance_generations = ["current", "previous"] - spot_max_price_percentage_over_lowest_price = 100 - - vcpu_count = { - min = 1 - } - - allowed_instance_types = ["t*", "m*"] - } - } - ] - } - } - } - metadata_options = { http_endpoint = "enabled" http_tokens = "required" From b745952ac805d3ff9ae79c0f312190f7e6f2bd25 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 30 Jul 2025 17:04:57 +0000 Subject: [PATCH 899/962] chore(release): version 21.0.6 [skip ci] ## [21.0.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.5...v21.0.6) (2025-07-30) ### Bug Fixes * Allow `instance_requirements` to be set in self-managed node groups ([#3455](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3455)) ([5322bf7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5322bf72fbbff4afb6a02ae283b21419d9de5b17)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be461948e3..1ba1615006 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.0.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.5...v21.0.6) (2025-07-30) + + +### Bug Fixes + +* Allow `instance_requirements` to be set in self-managed node groups ([#3455](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3455)) ([5322bf7](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/5322bf72fbbff4afb6a02ae283b21419d9de5b17)) + ## [21.0.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.4...v21.0.5) (2025-07-29) From 39be61d70232ba156fbf92ef90243b93fe5a9eee Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 2 Aug 2025 07:55:57 -0500 Subject: [PATCH 900/962] fix: Correct access policy logic to support not providing a policy to associate (#3464) --- README.md | 2 +- main.tf | 2 +- tests/eks-managed-node-group/main.tf | 6 ++++++ variables.tf | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fec86afc14..2b19625ef5 100644 --- a/README.md +++ b/README.md @@ -381,7 +381,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [access\_entries](#input\_access\_entries) | Map of access entries to add to the cluster |
map(object({
# Access entry
kubernetes_groups = optional(list(string))
principal_arn = string
type = optional(string, "STANDARD")
user_name = optional(string)
tags = optional(map(string), {})
# Access policy association
policy_associations = optional(map(object({
policy_arn = string
access_scope = object({
namespaces = optional(list(string))
type = string
})
})))
}))
| `{}` | no | +| [access\_entries](#input\_access\_entries) | Map of access entries to add to the cluster |
map(object({
# Access entry
kubernetes_groups = optional(list(string))
principal_arn = string
type = optional(string, "STANDARD")
user_name = optional(string)
tags = optional(map(string), {})
# Access policy association
policy_associations = optional(map(object({
policy_arn = string
access_scope = object({
namespaces = optional(list(string))
type = string
})
})), {})
}))
| `{}` | no | | [additional\_security\_group\_ids](#input\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | | [addons](#input\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` |
map(object({
name = optional(string) # will fall back to map key
before_compute = optional(bool, false)
most_recent = optional(bool, true)
addon_version = optional(string)
configuration_values = optional(string)
pod_identity_association = optional(list(object({
role_arn = string
service_account = string
})))
preserve = optional(bool, true)
resolve_conflicts_on_create = optional(string, "NONE")
resolve_conflicts_on_update = optional(string, "OVERWRITE")
service_account_role_arn = optional(string)
timeouts = optional(object({
create = optional(string)
update = optional(string)
delete = optional(string)
}))
tags = optional(map(string), {})
}))
| `null` | no | | [addons\_timeouts](#input\_addons\_timeouts) | Create, update, and delete timeout configurations for the cluster addons |
object({
create = optional(string)
update = optional(string)
delete = optional(string)
})
| `null` | no | diff --git a/main.tf b/main.tf index 48207dc238..fb443ddd3d 100644 --- a/main.tf +++ b/main.tf @@ -269,7 +269,7 @@ locals { # associations within a single entry flattened_access_entries = flatten([ for entry_key, entry_val in local.merged_access_entries : [ - for pol_key, pol_val in try(entry_val.policy_associations, {}) : + for pol_key, pol_val in entry_val.policy_associations : merge( { principal_arn = entry_val.principal_arn diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index 7d8adc08e3..27098981c4 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -408,6 +408,12 @@ module "eks" { } } } + + no-policy = { + kubernetes_groups = ["something"] + principal_arn = data.aws_caller_identity.current.arn + user_name = "someone" + } } tags = local.tags diff --git a/variables.tf b/variables.tf index 5e23df6c98..930a86e287 100644 --- a/variables.tf +++ b/variables.tf @@ -218,7 +218,7 @@ variable "access_entries" { namespaces = optional(list(string)) type = string }) - }))) + })), {}) })) default = {} } From eaee8fb8458a790641f6e81f5c3a60809b920589 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 2 Aug 2025 12:56:22 +0000 Subject: [PATCH 901/962] chore(release): version 21.0.7 [skip ci] ## [21.0.7](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.6...v21.0.7) (2025-08-02) ### Bug Fixes * Correct access policy logic to support not providing a policy to associate ([#3464](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3464)) ([39be61d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/39be61d70232ba156fbf92ef90243b93fe5a9eee)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ba1615006..a3ebd5d1d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.0.7](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.6...v21.0.7) (2025-08-02) + + +### Bug Fixes + +* Correct access policy logic to support not providing a policy to associate ([#3464](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3464)) ([39be61d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/39be61d70232ba156fbf92ef90243b93fe5a9eee)) + ## [21.0.6](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.5...v21.0.6) (2025-07-30) From ff1a6b73cbc426944e3b5d8dae39a4e85af0b600 Mon Sep 17 00:00:00 2001 From: Sergei Zhekpisov Date: Wed, 6 Aug 2025 15:05:51 +0100 Subject: [PATCH 902/962] docs: Add link to v21 upgrade guide in main `README.md` (#3468) Update README.md Add "Upgrade to v21.x" --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2b19625ef5..85dc75b9a1 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Terraform module which creates Amazon EKS (Kubernetes) resources - [Upgrade to v18.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-18.0.md) - [Upgrade to v19.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-19.0.md) - [Upgrade to v20.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-20.0.md) + - [Upgrade to v21.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-21.0.md) ### External Documentation From 6a887ad38686299c27333a83eb62310ed3106684 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 7 Aug 2025 14:49:54 -0500 Subject: [PATCH 903/962] fix: Allow disabling instance refresh on self-managed node groups (#3473) --- README.md | 2 +- modules/self-managed-node-group/README.md | 2 +- modules/self-managed-node-group/variables.tf | 3 +-- variables.tf | 10 +++++++++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 85dc75b9a1..e65a08f6db 100644 --- a/README.md +++ b/README.md @@ -472,7 +472,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [security\_group\_name](#input\_security\_group\_name) | Name to use on cluster security group created | `string` | `null` | no | | [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the cluster security group created | `map(string)` | `{}` | no | | [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix | `bool` | `true` | no | -| [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# Autoscaling Group
create_autoscaling_group = optional(bool)
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
availability_zones = optional(list(string))
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
desired_size_type = optional(string)
capacity_rebalance = optional(bool)
default_instance_warmup = optional(number)
protect_from_scale_in = optional(bool)
context = optional(string)
create_placement_group = optional(bool)
placement_group = optional(string)
health_check_type = optional(string)
health_check_grace_period = optional(number)
ignore_failed_scaling_activities = optional(bool)
force_delete = optional(bool)
termination_policies = optional(list(string))
suspended_processes = optional(list(string))
max_instance_lifetime = optional(number)
enabled_metrics = optional(list(string))
metrics_granularity = optional(string)
initial_lifecycle_hooks = optional(list(object({
default_result = optional(string)
heartbeat_timeout = optional(number)
lifecycle_transition = string
name = string
notification_metadata = optional(string)
notification_target_arn = optional(string)
role_arn = optional(string)
})))
instance_maintenance_policy = optional(object({
max_healthy_percentage = number
min_healthy_percentage = number
}))
instance_refresh = optional(object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string)
triggers = optional(list(string))
}))
use_mixed_instances_policy = optional(bool)
mixed_instances_policy = optional(object({
instances_distribution = optional(object({
on_demand_allocation_strategy = optional(string)
on_demand_base_capacity = optional(number)
on_demand_percentage_above_base_capacity = optional(number)
spot_allocation_strategy = optional(string)
spot_instance_pools = optional(number)
spot_max_price = optional(string)
}))
launch_template = object({
override = optional(list(object({
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = optional(number)
}))
}))
instance_type = optional(string)
launch_template_specification = optional(object({
launch_template_id = optional(string)
launch_template_name = optional(string)
version = optional(string)
}))
weighted_capacity = optional(string)
})))
})
}))
timeouts = optional(object({
delete = optional(string)
}))
autoscaling_group_tags = optional(map(string))
# User data
ami_type = optional(string)
additional_cluster_dns_ips = optional(list(string))
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
ami_id = optional(string)
instance_type = optional(string)
key_name = optional(string)
disable_api_termination = optional(bool)
instance_initiated_shutdown_behavior = optional(string)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = string
}))
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string))
subnet_id = optional(string)
})))
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_instance_profile = optional(bool)
iam_instance_profile_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Access entry
create_access_entry = optional(bool)
iam_role_arn = optional(string)
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | +| [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# Autoscaling Group
create_autoscaling_group = optional(bool)
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
availability_zones = optional(list(string))
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
desired_size_type = optional(string)
capacity_rebalance = optional(bool)
default_instance_warmup = optional(number)
protect_from_scale_in = optional(bool)
context = optional(string)
create_placement_group = optional(bool)
placement_group = optional(string)
health_check_type = optional(string)
health_check_grace_period = optional(number)
ignore_failed_scaling_activities = optional(bool)
force_delete = optional(bool)
termination_policies = optional(list(string))
suspended_processes = optional(list(string))
max_instance_lifetime = optional(number)
enabled_metrics = optional(list(string))
metrics_granularity = optional(string)
initial_lifecycle_hooks = optional(list(object({
default_result = optional(string)
heartbeat_timeout = optional(number)
lifecycle_transition = string
name = string
notification_metadata = optional(string)
notification_target_arn = optional(string)
role_arn = optional(string)
})))
instance_maintenance_policy = optional(object({
max_healthy_percentage = number
min_healthy_percentage = number
}))
instance_refresh = optional(object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string)
triggers = optional(list(string))
}),
# Default
{
strategy = "Rolling"
preferences = {
min_healthy_percentage = 66
}
}
)
use_mixed_instances_policy = optional(bool)
mixed_instances_policy = optional(object({
instances_distribution = optional(object({
on_demand_allocation_strategy = optional(string)
on_demand_base_capacity = optional(number)
on_demand_percentage_above_base_capacity = optional(number)
spot_allocation_strategy = optional(string)
spot_instance_pools = optional(number)
spot_max_price = optional(string)
}))
launch_template = object({
override = optional(list(object({
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = optional(number)
}))
}))
instance_type = optional(string)
launch_template_specification = optional(object({
launch_template_id = optional(string)
launch_template_name = optional(string)
version = optional(string)
}))
weighted_capacity = optional(string)
})))
})
}))
timeouts = optional(object({
delete = optional(string)
}))
autoscaling_group_tags = optional(map(string))
# User data
ami_type = optional(string)
additional_cluster_dns_ips = optional(list(string))
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
ami_id = optional(string)
instance_type = optional(string)
key_name = optional(string)
disable_api_termination = optional(bool)
instance_initiated_shutdown_behavior = optional(string)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = string
}))
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string))
subnet_id = optional(string)
})))
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_instance_profile = optional(bool)
iam_instance_profile_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Access entry
create_access_entry = optional(bool)
iam_role_arn = optional(string)
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | | [service\_ipv4\_cidr](#input\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [service\_ipv6\_cidr](#input\_service\_ipv6\_cidr) | The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster | `string` | `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets | `list(string)` | `[]` | no | diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 8b21f20b69..848dd4e654 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -144,7 +144,7 @@ module "self_managed_node_group" { | [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | | [instance\_maintenance\_policy](#input\_instance\_maintenance\_policy) | If this block is configured, add a instance maintenance policy to the specified Auto Scaling group |
object({
max_healthy_percentage = number
min_healthy_percentage = number
})
| `null` | no | | [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance |
object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
})
| `null` | no | -| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated |
object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number, 33)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string, "Rolling")
triggers = optional(list(string))
})
|
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | +| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated |
object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number, 66)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string, "Rolling")
triggers = optional(list(string))
})
|
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | | [instance\_requirements](#input\_instance\_requirements) | The attribute requirements for the type of instance. If present then `instance_type` cannot be present |
object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = string
}))
})
| `null` | no | | [instance\_type](#input\_instance\_type) | The type of the instance to launch | `string` | `"m6i.large"` | no | | [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index e3ef63e053..fd025948a7 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -710,7 +710,7 @@ variable "instance_refresh" { checkpoint_percentages = optional(list(number)) instance_warmup = optional(number) max_healthy_percentage = optional(number) - min_healthy_percentage = optional(number, 33) + min_healthy_percentage = optional(number, 66) scale_in_protected_instances = optional(string) skip_matching = optional(bool) standby_instances = optional(string) @@ -724,7 +724,6 @@ variable "instance_refresh" { min_healthy_percentage = 66 } } - nullable = false } variable "use_mixed_instances_policy" { diff --git a/variables.tf b/variables.tf index 930a86e287..7cf45f5118 100644 --- a/variables.tf +++ b/variables.tf @@ -841,7 +841,15 @@ variable "self_managed_node_groups" { })) strategy = optional(string) triggers = optional(list(string)) - })) + }), + # Default + { + strategy = "Rolling" + preferences = { + min_healthy_percentage = 66 + } + } + ) use_mixed_instances_policy = optional(bool) mixed_instances_policy = optional(object({ instances_distribution = optional(object({ From d97712a888093b0106651e1534d80f986fb67477 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 7 Aug 2025 19:50:18 +0000 Subject: [PATCH 904/962] chore(release): version 21.0.8 [skip ci] ## [21.0.8](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.7...v21.0.8) (2025-08-07) ### Bug Fixes * Allow disabling instance refresh on self-managed node groups ([#3473](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3473)) ([6a887ad](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6a887ad38686299c27333a83eb62310ed3106684)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3ebd5d1d1..fef2b930e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.0.8](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.7...v21.0.8) (2025-08-07) + + +### Bug Fixes + +* Allow disabling instance refresh on self-managed node groups ([#3473](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3473)) ([6a887ad](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/6a887ad38686299c27333a83eb62310ed3106684)) + ## [21.0.7](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.6...v21.0.7) (2025-08-02) From ca8f37e8ce2a15d0b216ac30e431fa4ac03fc8bc Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 13 Aug 2025 10:13:24 -0500 Subject: [PATCH 905/962] fix: Allow disabling instance refresh on self-managed node groups (part deux) (#3478) --- .pre-commit-config.yaml | 4 ++-- README.md | 2 +- modules/self-managed-node-group/README.md | 2 +- modules/self-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/variables.tf | 5 +++-- variables.tf | 9 +-------- 6 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b489ccee0d..7cf0036a18 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.99.5 + rev: v1.100.0 hooks: - id: terraform_fmt - id: terraform_docs @@ -23,7 +23,7 @@ repos: - '--args=--only=terraform_workspace_remote' - id: terraform_validate - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer diff --git a/README.md b/README.md index e65a08f6db..1fa4f292e9 100644 --- a/README.md +++ b/README.md @@ -472,7 +472,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [security\_group\_name](#input\_security\_group\_name) | Name to use on cluster security group created | `string` | `null` | no | | [security\_group\_tags](#input\_security\_group\_tags) | A map of additional tags to add to the cluster security group created | `map(string)` | `{}` | no | | [security\_group\_use\_name\_prefix](#input\_security\_group\_use\_name\_prefix) | Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix | `bool` | `true` | no | -| [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# Autoscaling Group
create_autoscaling_group = optional(bool)
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
availability_zones = optional(list(string))
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
desired_size_type = optional(string)
capacity_rebalance = optional(bool)
default_instance_warmup = optional(number)
protect_from_scale_in = optional(bool)
context = optional(string)
create_placement_group = optional(bool)
placement_group = optional(string)
health_check_type = optional(string)
health_check_grace_period = optional(number)
ignore_failed_scaling_activities = optional(bool)
force_delete = optional(bool)
termination_policies = optional(list(string))
suspended_processes = optional(list(string))
max_instance_lifetime = optional(number)
enabled_metrics = optional(list(string))
metrics_granularity = optional(string)
initial_lifecycle_hooks = optional(list(object({
default_result = optional(string)
heartbeat_timeout = optional(number)
lifecycle_transition = string
name = string
notification_metadata = optional(string)
notification_target_arn = optional(string)
role_arn = optional(string)
})))
instance_maintenance_policy = optional(object({
max_healthy_percentage = number
min_healthy_percentage = number
}))
instance_refresh = optional(object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string)
triggers = optional(list(string))
}),
# Default
{
strategy = "Rolling"
preferences = {
min_healthy_percentage = 66
}
}
)
use_mixed_instances_policy = optional(bool)
mixed_instances_policy = optional(object({
instances_distribution = optional(object({
on_demand_allocation_strategy = optional(string)
on_demand_base_capacity = optional(number)
on_demand_percentage_above_base_capacity = optional(number)
spot_allocation_strategy = optional(string)
spot_instance_pools = optional(number)
spot_max_price = optional(string)
}))
launch_template = object({
override = optional(list(object({
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = optional(number)
}))
}))
instance_type = optional(string)
launch_template_specification = optional(object({
launch_template_id = optional(string)
launch_template_name = optional(string)
version = optional(string)
}))
weighted_capacity = optional(string)
})))
})
}))
timeouts = optional(object({
delete = optional(string)
}))
autoscaling_group_tags = optional(map(string))
# User data
ami_type = optional(string)
additional_cluster_dns_ips = optional(list(string))
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
ami_id = optional(string)
instance_type = optional(string)
key_name = optional(string)
disable_api_termination = optional(bool)
instance_initiated_shutdown_behavior = optional(string)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = string
}))
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string))
subnet_id = optional(string)
})))
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_instance_profile = optional(bool)
iam_instance_profile_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Access entry
create_access_entry = optional(bool)
iam_role_arn = optional(string)
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | +| [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# Autoscaling Group
create_autoscaling_group = optional(bool)
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
availability_zones = optional(list(string))
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
desired_size_type = optional(string)
capacity_rebalance = optional(bool)
default_instance_warmup = optional(number)
protect_from_scale_in = optional(bool)
context = optional(string)
create_placement_group = optional(bool)
placement_group = optional(string)
health_check_type = optional(string)
health_check_grace_period = optional(number)
ignore_failed_scaling_activities = optional(bool)
force_delete = optional(bool)
termination_policies = optional(list(string))
suspended_processes = optional(list(string))
max_instance_lifetime = optional(number)
enabled_metrics = optional(list(string))
metrics_granularity = optional(string)
initial_lifecycle_hooks = optional(list(object({
default_result = optional(string)
heartbeat_timeout = optional(number)
lifecycle_transition = string
name = string
notification_metadata = optional(string)
notification_target_arn = optional(string)
role_arn = optional(string)
})))
instance_maintenance_policy = optional(object({
max_healthy_percentage = number
min_healthy_percentage = number
}))
instance_refresh = optional(object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string)
triggers = optional(list(string))
})
)
use_mixed_instances_policy = optional(bool)
mixed_instances_policy = optional(object({
instances_distribution = optional(object({
on_demand_allocation_strategy = optional(string)
on_demand_base_capacity = optional(number)
on_demand_percentage_above_base_capacity = optional(number)
spot_allocation_strategy = optional(string)
spot_instance_pools = optional(number)
spot_max_price = optional(string)
}))
launch_template = object({
override = optional(list(object({
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = optional(number)
}))
}))
instance_type = optional(string)
launch_template_specification = optional(object({
launch_template_id = optional(string)
launch_template_name = optional(string)
version = optional(string)
}))
weighted_capacity = optional(string)
})))
})
}))
timeouts = optional(object({
delete = optional(string)
}))
autoscaling_group_tags = optional(map(string))
# User data
ami_type = optional(string)
additional_cluster_dns_ips = optional(list(string))
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
ami_id = optional(string)
instance_type = optional(string)
key_name = optional(string)
disable_api_termination = optional(bool)
instance_initiated_shutdown_behavior = optional(string)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = string
}))
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string))
subnet_id = optional(string)
})))
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_instance_profile = optional(bool)
iam_instance_profile_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Access entry
create_access_entry = optional(bool)
iam_role_arn = optional(string)
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | | [service\_ipv4\_cidr](#input\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [service\_ipv6\_cidr](#input\_service\_ipv6\_cidr) | The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster | `string` | `null` | no | | [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets | `list(string)` | `[]` | no | diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 848dd4e654..2e30da46f5 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -144,7 +144,7 @@ module "self_managed_node_group" { | [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | | [instance\_maintenance\_policy](#input\_instance\_maintenance\_policy) | If this block is configured, add a instance maintenance policy to the specified Auto Scaling group |
object({
max_healthy_percentage = number
min_healthy_percentage = number
})
| `null` | no | | [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance |
object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
})
| `null` | no | -| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated |
object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number, 66)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string, "Rolling")
triggers = optional(list(string))
})
|
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | +| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated |
object({
preferences = optional(object({
alarm_specification = optional(object({
alarms = optional(list(string))
}))
auto_rollback = optional(bool)
checkpoint_delay = optional(number)
checkpoint_percentages = optional(list(number))
instance_warmup = optional(number)
max_healthy_percentage = optional(number)
min_healthy_percentage = optional(number)
scale_in_protected_instances = optional(string)
skip_matching = optional(bool)
standby_instances = optional(string)
}))
strategy = optional(string)
triggers = optional(list(string))
})
|
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | | [instance\_requirements](#input\_instance\_requirements) | The attribute requirements for the type of instance. If present then `instance_type` cannot be present |
object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = string
}))
})
| `null` | no | | [instance\_type](#input\_instance\_type) | The type of the instance to launch | `string` | `"m6i.large"` | no | | [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index b30476cbdf..d62bb88a02 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -526,7 +526,7 @@ resource "aws_autoscaling_group" "this" { } dynamic "instance_refresh" { - for_each = var.instance_refresh != null ? [var.instance_refresh] : [] + for_each = length({ for k, v in var.instance_refresh : k => v if v != null }) > 0 ? [var.instance_refresh] : [] content { dynamic "preferences" { diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index fd025948a7..a3168a6aa7 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -710,12 +710,12 @@ variable "instance_refresh" { checkpoint_percentages = optional(list(number)) instance_warmup = optional(number) max_healthy_percentage = optional(number) - min_healthy_percentage = optional(number, 66) + min_healthy_percentage = optional(number) scale_in_protected_instances = optional(string) skip_matching = optional(bool) standby_instances = optional(string) })) - strategy = optional(string, "Rolling") + strategy = optional(string) triggers = optional(list(string)) }) default = { @@ -724,6 +724,7 @@ variable "instance_refresh" { min_healthy_percentage = 66 } } + nullable = false } variable "use_mixed_instances_policy" { diff --git a/variables.tf b/variables.tf index 7cf45f5118..cf238070b6 100644 --- a/variables.tf +++ b/variables.tf @@ -841,14 +841,7 @@ variable "self_managed_node_groups" { })) strategy = optional(string) triggers = optional(list(string)) - }), - # Default - { - strategy = "Rolling" - preferences = { - min_healthy_percentage = 66 - } - } + }) ) use_mixed_instances_policy = optional(bool) mixed_instances_policy = optional(object({ From 30664d261760ec97c670f90df3c793431cb096fb Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 13 Aug 2025 15:13:48 +0000 Subject: [PATCH 906/962] chore(release): version 21.0.9 [skip ci] ## [21.0.9](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.8...v21.0.9) (2025-08-13) ### Bug Fixes * Allow disabling instance refresh on self-managed node groups (part deux) ([#3478](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3478)) ([ca8f37e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ca8f37e8ce2a15d0b216ac30e431fa4ac03fc8bc)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fef2b930e9..fa46f327b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.0.9](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.8...v21.0.9) (2025-08-13) + + +### Bug Fixes + +* Allow disabling instance refresh on self-managed node groups (part deux) ([#3478](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3478)) ([ca8f37e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/ca8f37e8ce2a15d0b216ac30e431fa4ac03fc8bc)) + ## [21.0.8](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.7...v21.0.8) (2025-08-07) From 83c9cd187a36c10f46472e82a197212e897f7f0d Mon Sep 17 00:00:00 2001 From: Tiago Vieira <80104680+tiagovmvieira@users.noreply.github.com> Date: Fri, 15 Aug 2025 13:23:13 +0100 Subject: [PATCH 907/962] feat: Add support for deletion protection functionality in the cluster (#3475) * feat: Add support for deletion protection functionality in the cluster * fix: Bump version throughout, correct argument/variable name --------- Co-authored-by: Tiago Vieira Co-authored-by: Bryant Biggs --- README.md | 5 +++-- examples/eks-auto-mode/README.md | 4 ++-- examples/eks-auto-mode/main.tf | 1 + examples/eks-auto-mode/versions.tf | 2 +- examples/eks-hybrid-nodes/README.md | 6 +++--- examples/eks-hybrid-nodes/versions.tf | 2 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 10 +++++----- examples/karpenter/versions.tf | 4 ++-- examples/self-managed-node-group/versions.tf | 2 +- main.tf | 1 + modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/hybrid-node-role/README.md | 4 ++-- modules/hybrid-node-role/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-fargate-profile/README.md | 4 ++-- tests/eks-fargate-profile/versions.tf | 2 +- tests/eks-hybrid-nodes/README.md | 2 +- tests/eks-hybrid-nodes/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- variables.tf | 6 ++++++ versions.tf | 2 +- 31 files changed, 54 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 1fa4f292e9..473999e4bc 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 4.0 | @@ -326,7 +326,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 4.0 | @@ -406,6 +406,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [create\_security\_group](#input\_create\_security\_group) | Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default | `bool` | `true` | no | | [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | | [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed node group(s), self-managed node group(s), Fargate profile(s)) | `string` | `"30s"` | no | +| [deletion\_protection](#input\_deletion\_protection) | Whether to enable deletion protection for the cluster. When enabled, the cluster cannot be deleted unless deletion protection is first disabled | `bool` | `null` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# EKS Managed Node Group
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
ami_id = optional(string)
ami_type = optional(string)
ami_release_version = optional(string)
use_latest_ami_release_version = optional(bool)
capacity_type = optional(string)
disk_size = optional(number)
force_update_version = optional(bool)
instance_types = optional(list(string))
labels = optional(map(string))
node_repair_config = optional(object({
enabled = optional(bool)
}))
remote_access = optional(object({
ec2_ssh_key = optional(string)
source_security_group_ids = optional(list(string))
}))
taints = optional(map(object({
key = string
value = optional(string)
effect = string
})))
update_config = optional(object({
max_unavailable = optional(number)
max_unavailable_percentage = optional(number)
}))
timeouts = optional(object({
create = optional(string)
update = optional(string)
delete = optional(string)
}))
# User data
enable_bootstrap_user_data = optional(bool)
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
key_name = optional(string)
disable_api_termination = optional(bool)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
create_placement_group = optional(bool)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
subnet_id = optional(string)
})))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_role = optional(bool)
iam_role_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
cluster_primary_security_group_id = optional(string)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})), {})
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | | [enable\_auto\_mode\_custom\_tags](#input\_enable\_auto\_mode\_custom\_tags) | Determines whether to enable permissions for custom tags resources created by EKS Auto Mode | `bool` | `true` | no | | [enable\_cluster\_creator\_admin\_permissions](#input\_enable\_cluster\_creator\_admin\_permissions) | Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry | `bool` | `false` | no | diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index 57da18f875..204fecd0b4 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -25,13 +25,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | ## Modules diff --git a/examples/eks-auto-mode/main.tf b/examples/eks-auto-mode/main.tf index c99c60c970..e00b825f63 100644 --- a/examples/eks-auto-mode/main.tf +++ b/examples/eks-auto-mode/main.tf @@ -35,6 +35,7 @@ module "eks" { name = local.name kubernetes_version = local.kubernetes_version endpoint_public_access = true + deletion_protection = true enable_cluster_creator_admin_permissions = true diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index 94ccb4eb4d..d2b9e1c244 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -26,7 +26,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | | [helm](#requirement\_helm) | >= 3.0 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | @@ -36,8 +36,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | -| [aws.remote](#provider\_aws.remote) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | +| [aws.remote](#provider\_aws.remote) | >= 6.9 | | [helm](#provider\_helm) | >= 3.0 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index 29273c9ce0..8762781644 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } helm = { source = "hashicorp/helm" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 47cfbf3ae2..8e7e11ecf4 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -94,16 +94,16 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | -| [helm](#requirement\_helm) | >= 3.0.0 | +| [aws](#requirement\_aws) | >= 6.9 | +| [helm](#requirement\_helm) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | -| [aws.virginia](#provider\_aws.virginia) | >= 6.0 | -| [helm](#provider\_helm) | >= 3.0.0 | +| [aws](#provider\_aws) | >= 6.9 | +| [aws.virginia](#provider\_aws.virginia) | >= 6.9 | +| [helm](#provider\_helm) | >= 3.0 | ## Modules diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 4e81032c58..6cd7d576fc 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,11 +4,11 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } helm = { source = "hashicorp/helm" - version = ">= 3.0.0" + version = ">= 3.0" } } } diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/main.tf b/main.tf index fb443ddd3d..d66cf3d948 100644 --- a/main.tf +++ b/main.tf @@ -42,6 +42,7 @@ resource "aws_eks_cluster" "this" { role_arn = local.role_arn version = var.kubernetes_version enabled_cluster_log_types = var.enabled_log_types + deletion_protection = var.deletion_protection bootstrap_self_managed_addons = false force_update_version = var.force_update_version diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 05eacc433c..00dc6f71f2 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 61ef2b980f..8981d2a4f9 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index 176aabcdf4..9e7e0f4570 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -75,13 +75,13 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | ## Modules diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 85e472b225..36f3412fe9 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 2e30da46f5..d834aa210d 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index 2696041cea..5ff2f7f79c 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | ## Modules diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index efe26912ce..27875d930e 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -18,7 +18,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index 3c2ec900b1..d56b124e91 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 15a05fa9e6..4b69f395b7 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index feee069124..eba5c2f79b 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.0 | +| [aws](#requirement\_aws) | >= 6.9 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.0 | +| [aws](#provider\_aws) | >= 6.9 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index db13b0a8d2..fe4021c7cf 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } } } diff --git a/variables.tf b/variables.tf index cf238070b6..a9d2b4342c 100644 --- a/variables.tf +++ b/variables.tf @@ -44,6 +44,12 @@ variable "enabled_log_types" { default = ["audit", "api", "authenticator"] } +variable "deletion_protection" { + description = "Whether to enable deletion protection for the cluster. When enabled, the cluster cannot be deleted unless deletion protection is first disabled" + type = bool + default = null +} + variable "force_update_version" { description = "Force version update by overriding upgrade-blocking readiness checks when updating a cluster" type = bool diff --git a/versions.tf b/versions.tf index 71efea2e03..6c8090303a 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = ">= 6.9" } tls = { source = "hashicorp/tls" From 82acedc0593723979dc653d726617884cf7ab447 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 15 Aug 2025 12:23:39 +0000 Subject: [PATCH 908/962] chore(release): version 21.1.0 [skip ci] ## [21.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.9...v21.1.0) (2025-08-15) ### Features * Add support for deletion protection functionality in the cluster ([#3475](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3475)) ([83c9cd1](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/83c9cd187a36c10f46472e82a197212e897f7f0d)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa46f327b2..db9cdb26aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.9...v21.1.0) (2025-08-15) + + +### Features + +* Add support for deletion protection functionality in the cluster ([#3475](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3475)) ([83c9cd1](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/83c9cd187a36c10f46472e82a197212e897f7f0d)) + ## [21.0.9](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.8...v21.0.9) (2025-08-13) From b40968a503f1134adcb986af9b4c7f3f3514b811 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 24 Aug 2025 10:08:35 -0500 Subject: [PATCH 909/962] fix: Correct metadata options loop condition due to variable definition defaults (#3490) --- modules/eks-managed-node-group/main.tf | 2 +- modules/self-managed-node-group/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index d07c38633d..1e8b8f1508 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -224,7 +224,7 @@ resource "aws_launch_template" "this" { } dynamic "metadata_options" { - for_each = var.metadata_options != null ? [var.metadata_options] : [] + for_each = [var.metadata_options] content { http_endpoint = metadata_options.value.http_endpoint diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index d62bb88a02..7ee3f40e84 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -342,7 +342,7 @@ resource "aws_launch_template" "this" { } dynamic "metadata_options" { - for_each = var.metadata_options != null ? [var.metadata_options] : [] + for_each = [var.metadata_options] content { http_endpoint = metadata_options.value.http_endpoint From f56e87eecd955f66be92a1ba02b9e9fa82be43b6 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 24 Aug 2025 15:09:03 +0000 Subject: [PATCH 910/962] chore(release): version 21.1.1 [skip ci] ## [21.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.0...v21.1.1) (2025-08-24) ### Bug Fixes * Correct metadata options loop condition due to variable definition defaults ([#3490](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3490)) ([b40968a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b40968a503f1134adcb986af9b4c7f3f3514b811)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index db9cdb26aa..9868d219c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.0...v21.1.1) (2025-08-24) + + +### Bug Fixes + +* Correct metadata options loop condition due to variable definition defaults ([#3490](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3490)) ([b40968a](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b40968a503f1134adcb986af9b4c7f3f3514b811)) + ## [21.1.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.0.9...v21.1.0) (2025-08-15) From 3959b65672286c84c03012e12a2e7c8630db6c11 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 24 Aug 2025 12:59:13 -0500 Subject: [PATCH 911/962] fix: Remediate type mismatch for EFA interfaces and ensure correct (local) definition is used (#3491) --- modules/eks-managed-node-group/README.md | 2 +- modules/eks-managed-node-group/main.tf | 44 +++++++++++++++++--- modules/eks-managed-node-group/variables.tf | 1 - modules/self-managed-node-group/README.md | 2 +- modules/self-managed-node-group/main.tf | 44 +++++++++++++++++--- modules/self-managed-node-group/variables.tf | 1 - tests/eks-managed-node-group/main.tf | 17 ++++---- tests/self-managed-node-group/main.tf | 17 ++++---- 8 files changed, 92 insertions(+), 36 deletions(-) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 00dc6f71f2..6c923884f9 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -169,7 +169,7 @@ module "eks_managed_node_group" { | [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance |
object({
http_endpoint = optional(string, "enabled")
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number, 1)
http_tokens = optional(string, "required")
instance_metadata_tags = optional(string)
})
|
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 1,
"http_tokens": "required"
}
| no | | [min\_size](#input\_min\_size) | Minimum number of instances/nodes | `number` | `1` | no | | [name](#input\_name) | Name of the EKS managed node group | `string` | `""` | no | -| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time |
list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
subnet_id = optional(string)
}))
| `[]` | no | +| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time |
list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
}))
| `[]` | no | | [node\_repair\_config](#input\_node\_repair\_config) | The node auto repair configuration for the node group |
object({
enabled = optional(bool, true)
})
| `null` | no | | [partition](#input\_partition) | The AWS partition - pass through value to reduce number of GET requests from data sources | `string` | `""` | no | | [placement](#input\_placement) | The placement of the instance |
object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
})
| `null` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 1e8b8f1508..85d75efece 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -41,7 +41,7 @@ module "user_data" { ################################################################################ data "aws_ec2_instance_type" "this" { - count = var.create && var.enable_efa_support ? 1 : 0 + count = local.enable_efa_support ? 1 : 0 region = var.region @@ -62,6 +62,24 @@ locals { device_index = i == 0 ? 0 : 1 network_card_index = i interface_type = var.enable_efa_only ? contains(concat([0], var.efa_indices), i) ? "efa" : "efa-only" : "efa" + + # Null out due to error: The true and false result expressions must have consistent types. The 'true' value is tuple, but the 'false' value is list of objects. + associate_carrier_ip_address = null + connection_tracking_specification = null + description = "EFA${var.enable_efa_only ? "-only" : ""} Network Interface ${i}" + ena_srd_specification = null + ipv4_address_count = null + ipv4_addresses = null + ipv4_prefix_count = null + ipv4_prefixes = null + ipv6_address_count = null + ipv6_addresses = null + ipv6_prefix_count = null + ipv6_prefixes = null + network_interface_id = null + primary_ipv6 = null + private_ip_address = null + security_groups = [] } ] @@ -247,7 +265,7 @@ resource "aws_launch_template" "this" { name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null dynamic "network_interfaces" { - for_each = length(var.network_interfaces) > 0 ? var.network_interfaces : [] + for_each = length(local.network_interfaces) > 0 ? local.network_interfaces : [] content { associate_carrier_ip_address = network_interfaces.value.associate_carrier_ip_address @@ -688,9 +706,16 @@ locals { { all_self_efa = { description = "Node to node EFA" - protocol = "-1" - from_port = 0 + ip_protocol = "-1" self = true + + # Null out due to variable type and not using `try()` in resource + cidr_ipv4 = null + cidr_ipv6 = null + from_port = null + name = null + prefix_list_id = null + tags = {} } } : k => v if var.enable_efa_support }, @@ -700,9 +725,16 @@ locals { { all_self_efa = { description = "Node to node EFA" - protocol = "-1" - to_port = 0 + ip_protocol = "-1" self = true + + # Null out due to variable type and not using `try()` in resource + cidr_ipv4 = null + cidr_ipv6 = null + to_port = null + name = null + prefix_list_id = null + tags = {} } } : k => v if var.enable_efa_support }, diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 6e65933e51..84bb8421d2 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -384,7 +384,6 @@ variable "network_interfaces" { primary_ipv6 = optional(bool) private_ip_address = optional(string) security_groups = optional(list(string), []) - subnet_id = optional(string) })) default = [] nullable = false diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index d834aa210d..c14f190d47 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -166,7 +166,7 @@ module "self_managed_node_group" { | [min\_size](#input\_min\_size) | The minimum size of the autoscaling group | `number` | `1` | no | | [mixed\_instances\_policy](#input\_mixed\_instances\_policy) | Configuration block containing settings to define launch targets for Auto Scaling groups |
object({
instances_distribution = optional(object({
on_demand_allocation_strategy = optional(string)
on_demand_base_capacity = optional(number)
on_demand_percentage_above_base_capacity = optional(number)
spot_allocation_strategy = optional(string)
spot_instance_pools = optional(number)
spot_max_price = optional(string)
}))
launch_template = object({
override = optional(list(object({
instance_requirements = optional(object({
accelerator_count = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_manufacturers = optional(list(string))
accelerator_names = optional(list(string))
accelerator_total_memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
accelerator_types = optional(list(string))
allowed_instance_types = optional(list(string))
bare_metal = optional(string)
baseline_ebs_bandwidth_mbps = optional(object({
max = optional(number)
min = optional(number)
}))
burstable_performance = optional(string)
cpu_manufacturers = optional(list(string))
excluded_instance_types = optional(list(string))
instance_generations = optional(list(string))
local_storage = optional(string)
local_storage_types = optional(list(string))
max_spot_price_as_percentage_of_optimal_on_demand_price = optional(number)
memory_gib_per_vcpu = optional(object({
max = optional(number)
min = optional(number)
}))
memory_mib = optional(object({
max = optional(number)
min = optional(number)
}))
network_bandwidth_gbps = optional(object({
max = optional(number)
min = optional(number)
}))
network_interface_count = optional(object({
max = optional(number)
min = optional(number)
}))
on_demand_max_price_percentage_over_lowest_price = optional(number)
require_hibernate_support = optional(bool)
spot_max_price_percentage_over_lowest_price = optional(number)
total_local_storage_gb = optional(object({
max = optional(number)
min = optional(number)
}))
vcpu_count = optional(object({
max = optional(number)
min = optional(number)
}))
}))
instance_type = optional(string)
launch_template_specification = optional(object({
launch_template_id = optional(string)
launch_template_name = optional(string)
version = optional(string)
}))
weighted_capacity = optional(string)
})))
})
})
| `null` | no | | [name](#input\_name) | Name of the Self managed Node Group | `string` | `""` | no | -| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time |
list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
subnet_id = optional(string)
}))
| `[]` | no | +| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time |
list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
}))
| `[]` | no | | [partition](#input\_partition) | The AWS partition - pass through value to reduce number of GET requests from data sources | `string` | `""` | no | | [placement](#input\_placement) | The placement of the instance |
object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
})
| `null` | no | | [placement\_group](#input\_placement\_group) | The name of the placement group into which you'll launch your instances | `string` | `null` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index 7ee3f40e84..c221521de4 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -82,7 +82,7 @@ module "user_data" { ################################################################################ data "aws_ec2_instance_type" "this" { - count = var.create && var.enable_efa_support ? 1 : 0 + count = local.enable_efa_support ? 1 : 0 region = var.region @@ -103,6 +103,24 @@ locals { device_index = i == 0 ? 0 : 1 network_card_index = i interface_type = var.enable_efa_only ? contains(concat([0], var.efa_indices), i) ? "efa" : "efa-only" : "efa" + + # Null out due to error: The true and false result expressions must have consistent types. The 'true' value is tuple, but the 'false' value is list of objects. + associate_carrier_ip_address = null + connection_tracking_specification = null + description = "EFA${var.enable_efa_only ? "-only" : ""} Network Interface ${i}" + ena_srd_specification = null + ipv4_address_count = null + ipv4_addresses = null + ipv4_prefix_count = null + ipv4_prefixes = null + ipv6_address_count = null + ipv6_addresses = null + ipv6_prefix_count = null + ipv6_prefixes = null + network_interface_id = null + primary_ipv6 = null + private_ip_address = null + security_groups = [] } ] @@ -365,7 +383,7 @@ resource "aws_launch_template" "this" { name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null dynamic "network_interfaces" { - for_each = length(var.network_interfaces) > 0 ? var.network_interfaces : [] + for_each = length(local.network_interfaces) > 0 ? local.network_interfaces : [] content { associate_carrier_ip_address = network_interfaces.value.associate_carrier_ip_address @@ -976,9 +994,16 @@ locals { { all_self_efa = { description = "Node to node EFA" - protocol = "-1" - from_port = 0 + ip_protocol = "-1" self = true + + # Null out due to variable type and not using `try()` in resource + cidr_ipv4 = null + cidr_ipv6 = null + from_port = null + name = null + prefix_list_id = null + tags = {} } } : k => v if var.enable_efa_support }, @@ -988,9 +1013,16 @@ locals { { all_self_efa = { description = "Node to node EFA" - protocol = "-1" - to_port = 0 + ip_protocol = "-1" self = true + + # Null out due to variable type and not using `try()` in resource + cidr_ipv4 = null + cidr_ipv6 = null + to_port = null + name = null + prefix_list_id = null + tags = {} } } : k => v if var.enable_efa_support }, diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index a3168a6aa7..6e42508c68 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -311,7 +311,6 @@ variable "network_interfaces" { primary_ipv6 = optional(bool) private_ip_address = optional(string) security_groups = optional(list(string), []) - subnet_id = optional(string) })) default = [] nullable = false diff --git a/tests/eks-managed-node-group/main.tf b/tests/eks-managed-node-group/main.tf index 27098981c4..c09bde0a3b 100644 --- a/tests/eks-managed-node-group/main.tf +++ b/tests/eks-managed-node-group/main.tf @@ -318,14 +318,15 @@ module "eks" { } efa = { - # Disabling automatic creation due to instance type/quota availability - # Can be enabled when appropriate for testing/validation - create = false - # The EKS AL2023 NVIDIA AMI provides all of the necessary components # for accelerated workloads w/ EFA ami_type = "AL2023_x86_64_NVIDIA" - instance_types = ["p5e.48xlarge"] + instance_types = ["p4d.24xlarge"] + + # Setting to zero so all resources are created *EXCEPT the EC2 instances + min_size = 0 + max_size = 1 + desired_size = 0 # Mount instance store volumes in RAID-0 for kubelet and containerd # https://github.com/awslabs/amazon-eks-ami/blob/master/doc/USER_GUIDE.md#raid-0-for-kubelet-and-containerd-raid0 @@ -350,11 +351,7 @@ module "eks" { # 3. Expose all of the available EFA interfaces on the launch template enable_efa_support = true enable_efa_only = true - efa_indices = [0, 4, 8, 12] - - min_size = 1 - max_size = 1 - desired_size = 1 + efa_indices = [0] labels = { "vpc.amazonaws.com/efa.present" = "true" diff --git a/tests/self-managed-node-group/main.tf b/tests/self-managed-node-group/main.tf index 9fc3fdc6b1..50611d4b3d 100644 --- a/tests/self-managed-node-group/main.tf +++ b/tests/self-managed-node-group/main.tf @@ -309,14 +309,15 @@ module "eks" { } efa = { - # Disabling automatic creation due to instance type/quota availability - # Can be enabled when appropriate for testing/validation - create = false - # The EKS AL2023 NVIDIA AMI provides all of the necessary components # for accelerated workloads w/ EFA ami_type = "AL2023_x86_64_NVIDIA" - instance_types = ["p5e.48xlarge"] + instance_types = ["p4d.24xlarge"] + + # Setting to zero so all resources are created *EXCEPT the EC2 instances + min_size = 0 + max_size = 1 + desired_size = 0 # Mount instance store volumes in RAID-0 for kubelet and containerd # https://github.com/awslabs/amazon-eks-ami/blob/master/doc/USER_GUIDE.md#raid-0-for-kubelet-and-containerd-raid0 @@ -341,11 +342,7 @@ module "eks" { # 3. Expose all of the available EFA interfaces on the launch template enable_efa_support = true enable_efa_only = true - efa_indices = [0, 4, 8, 12] - - min_size = 2 - max_size = 2 - desired_size = 2 + efa_indices = [0] labels = { "vpc.amazonaws.com/efa.present" = "true" From b02727a652630ecda59776c91ad748c4b14c302d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 24 Aug 2025 17:59:44 +0000 Subject: [PATCH 912/962] chore(release): version 21.1.2 [skip ci] ## [21.1.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.1...v21.1.2) (2025-08-24) ### Bug Fixes * Remediate type mismatch for EFA interfaces and ensure correct (local) definition is used ([#3491](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3491)) ([3959b65](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3959b65672286c84c03012e12a2e7c8630db6c11)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9868d219c9..1306f634b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.1.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.1...v21.1.2) (2025-08-24) + + +### Bug Fixes + +* Remediate type mismatch for EFA interfaces and ensure correct (local) definition is used ([#3491](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3491)) ([3959b65](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/3959b65672286c84c03012e12a2e7c8630db6c11)) + ## [21.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.0...v21.1.1) (2025-08-24) From b236208d5ce9ff14447f3d8d580b71790c8074e9 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 24 Aug 2025 13:16:17 -0500 Subject: [PATCH 913/962] fix: Correct addon timeout lookup/override logic to support global and addon specific settings (#3492) --- README.md | 4 ++-- main.tf | 12 ++++++------ variables.tf | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 473999e4bc..ec22fdd351 100644 --- a/README.md +++ b/README.md @@ -384,8 +384,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|-------------|------|---------|:--------:| | [access\_entries](#input\_access\_entries) | Map of access entries to add to the cluster |
map(object({
# Access entry
kubernetes_groups = optional(list(string))
principal_arn = string
type = optional(string, "STANDARD")
user_name = optional(string)
tags = optional(map(string), {})
# Access policy association
policy_associations = optional(map(object({
policy_arn = string
access_scope = object({
namespaces = optional(list(string))
type = string
})
})), {})
}))
| `{}` | no | | [additional\_security\_group\_ids](#input\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no | -| [addons](#input\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` |
map(object({
name = optional(string) # will fall back to map key
before_compute = optional(bool, false)
most_recent = optional(bool, true)
addon_version = optional(string)
configuration_values = optional(string)
pod_identity_association = optional(list(object({
role_arn = string
service_account = string
})))
preserve = optional(bool, true)
resolve_conflicts_on_create = optional(string, "NONE")
resolve_conflicts_on_update = optional(string, "OVERWRITE")
service_account_role_arn = optional(string)
timeouts = optional(object({
create = optional(string)
update = optional(string)
delete = optional(string)
}))
tags = optional(map(string), {})
}))
| `null` | no | -| [addons\_timeouts](#input\_addons\_timeouts) | Create, update, and delete timeout configurations for the cluster addons |
object({
create = optional(string)
update = optional(string)
delete = optional(string)
})
| `null` | no | +| [addons](#input\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` |
map(object({
name = optional(string) # will fall back to map key
before_compute = optional(bool, false)
most_recent = optional(bool, true)
addon_version = optional(string)
configuration_values = optional(string)
pod_identity_association = optional(list(object({
role_arn = string
service_account = string
})))
preserve = optional(bool, true)
resolve_conflicts_on_create = optional(string, "NONE")
resolve_conflicts_on_update = optional(string, "OVERWRITE")
service_account_role_arn = optional(string)
timeouts = optional(object({
create = optional(string)
update = optional(string)
delete = optional(string)
}), {})
tags = optional(map(string), {})
}))
| `null` | no | +| [addons\_timeouts](#input\_addons\_timeouts) | Create, update, and delete timeout configurations for the cluster addons |
object({
create = optional(string)
update = optional(string)
delete = optional(string)
})
| `{}` | no | | [attach\_encryption\_policy](#input\_attach\_encryption\_policy) | Indicates whether or not to attach an additional policy for the cluster IAM role to utilize the encryption key provided | `bool` | `true` | no | | [authentication\_mode](#input\_authentication\_mode) | The authentication mode for the cluster. Valid values are `CONFIG_MAP`, `API` or `API_AND_CONFIG_MAP` | `string` | `"API_AND_CONFIG_MAP"` | no | | [cloudwatch\_log\_group\_class](#input\_cloudwatch\_log\_group\_class) | Specified the log class of the log group. Possible values are: `STANDARD` or `INFREQUENT_ACCESS` | `string` | `null` | no | diff --git a/main.tf b/main.tf index d66cf3d948..8ea8629b8d 100644 --- a/main.tf +++ b/main.tf @@ -785,9 +785,9 @@ resource "aws_eks_addon" "this" { service_account_role_arn = each.value.service_account_role_arn timeouts { - create = try(coalesce(each.value.timeouts.create, var.addons_timeouts.create), null) - update = try(coalesce(each.value.timeouts.update, var.addons_timeouts.update), null) - delete = try(coalesce(each.value.timeouts.delete, var.addons_timeouts.delete), null) + create = each.value.timeouts.create != null ? each.value.timeouts.create : var.addons_timeouts.create + update = each.value.timeouts.update != null ? each.value.timeouts.update : var.addons_timeouts.update + delete = each.value.timeouts.delete != null ? each.value.timeouts.delete : var.addons_timeouts.delete } tags = merge( @@ -830,9 +830,9 @@ resource "aws_eks_addon" "before_compute" { service_account_role_arn = each.value.service_account_role_arn timeouts { - create = try(coalesce(each.value.timeouts.create, var.addons_timeouts.create), null) - update = try(coalesce(each.value.timeouts.update, var.addons_timeouts.update), null) - delete = try(coalesce(each.value.timeouts.delete, var.addons_timeouts.delete), null) + create = each.value.timeouts.create != null ? each.value.timeouts.create : var.addons_timeouts.create + update = each.value.timeouts.update != null ? each.value.timeouts.update : var.addons_timeouts.update + delete = each.value.timeouts.delete != null ? each.value.timeouts.delete : var.addons_timeouts.delete } tags = merge( diff --git a/variables.tf b/variables.tf index a9d2b4342c..9647017980 100644 --- a/variables.tf +++ b/variables.tf @@ -635,7 +635,7 @@ variable "addons" { create = optional(string) update = optional(string) delete = optional(string) - })) + }), {}) tags = optional(map(string), {}) })) default = null @@ -648,7 +648,7 @@ variable "addons_timeouts" { update = optional(string) delete = optional(string) }) - default = null + default = {} } ################################################################################ From e5c35cfd4edcdfc5653c813a0ec18743a3160d95 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 24 Aug 2025 18:16:49 +0000 Subject: [PATCH 914/962] chore(release): version 21.1.3 [skip ci] ## [21.1.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.2...v21.1.3) (2025-08-24) ### Bug Fixes * Correct addon timeout lookup/override logic to support global and addon specific settings ([#3492](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3492)) ([b236208](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b236208d5ce9ff14447f3d8d580b71790c8074e9)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1306f634b0..c90d957414 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.1.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.2...v21.1.3) (2025-08-24) + + +### Bug Fixes + +* Correct addon timeout lookup/override logic to support global and addon specific settings ([#3492](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3492)) ([b236208](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/b236208d5ce9ff14447f3d8d580b71790c8074e9)) + ## [21.1.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.1...v21.1.2) (2025-08-24) From e5cff842835f2bdede53db843c2b37b3d3534332 Mon Sep 17 00:00:00 2001 From: Kamontat Chantrachirathumrong <14089557+kamontat@users.noreply.github.com> Date: Tue, 26 Aug 2025 01:21:39 +0700 Subject: [PATCH 915/962] fix: Ensure module created security group is included on any network interfaces created (#3493) * fix(module): wrong variable name in eks-managed-node-group * Update modules/eks-managed-node-group/main.tf --------- Co-authored-by: Bryant Biggs --- modules/eks-managed-node-group/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 85d75efece..fa940c51a9 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -315,7 +315,7 @@ resource "aws_launch_template" "this" { primary_ipv6 = network_interfaces.value.primary_ipv6 private_ip_address = network_interfaces.value.private_ip_address # Ref: https://github.com/hashicorp/terraform-provider-aws/issues/4570 - security_groups = compact(concat(network_interfaces.value.security_groups, var.vpc_security_group_ids)) + security_groups = compact(concat(network_interfaces.value.security_groups, local.security_group_ids)) # Set on EKS managed node group, will fail if set here # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics # subnet_id = try(network_interfaces.value.subnet_id, null) From 23a67d06d782fd6246f5616931c484afc0bc03e6 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 25 Aug 2025 18:22:13 +0000 Subject: [PATCH 916/962] chore(release): version 21.1.4 [skip ci] ## [21.1.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.3...v21.1.4) (2025-08-25) ### Bug Fixes * Ensure module created security group is included on any network interfaces created ([#3493](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3493)) ([e5cff84](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e5cff842835f2bdede53db843c2b37b3d3534332)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c90d957414..b2a12ff423 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.1.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.3...v21.1.4) (2025-08-25) + + +### Bug Fixes + +* Ensure module created security group is included on any network interfaces created ([#3493](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3493)) ([e5cff84](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e5cff842835f2bdede53db843c2b37b3d3534332)) + ## [21.1.3](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.2...v21.1.3) (2025-08-24) From fa1d4221c8fd346927e88d617181fdb75790ecf8 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 25 Aug 2025 20:10:47 -0500 Subject: [PATCH 917/962] fix: Ensure module created security group is included on any network interfaces created (#3495) --- modules/self-managed-node-group/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index c221521de4..8a65dc6a66 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -433,7 +433,7 @@ resource "aws_launch_template" "this" { primary_ipv6 = network_interfaces.value.primary_ipv6 private_ip_address = network_interfaces.value.private_ip_address # Ref: https://github.com/hashicorp/terraform-provider-aws/issues/4570 - security_groups = compact(concat(network_interfaces.value.security_groups, var.vpc_security_group_ids)) + security_groups = compact(concat(network_interfaces.value.security_groups, local.security_group_ids)) # Set on EKS managed node group, will fail if set here # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics # subnet_id = try(network_interfaces.value.subnet_id, null) From b9e7ad0cccbfe2f076f05fc46eecd0dfaabbd17e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 26 Aug 2025 01:11:16 +0000 Subject: [PATCH 918/962] chore(release): version 21.1.5 [skip ci] ## [21.1.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.4...v21.1.5) (2025-08-26) ### Bug Fixes * Ensure module created security group is included on any network interfaces created ([#3495](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3495)) ([fa1d422](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/fa1d4221c8fd346927e88d617181fdb75790ecf8)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2a12ff423..b28440f1c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.1.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.4...v21.1.5) (2025-08-26) + + +### Bug Fixes + +* Ensure module created security group is included on any network interfaces created ([#3495](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3495)) ([fa1d422](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/fa1d4221c8fd346927e88d617181fdb75790ecf8)) + ## [21.1.4](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.3...v21.1.4) (2025-08-25) From 131db3973f7eaf539c33b73014058a94ac0d0528 Mon Sep 17 00:00:00 2001 From: Erez Zarum Date: Thu, 11 Sep 2025 16:27:45 +0300 Subject: [PATCH 919/962] feat: Update Karpenter controller policy and permissions to match upstream project (#3510) --- modules/karpenter/main.tf | 2 +- modules/karpenter/policy.tf | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 7eba2cb5d8..227a2f6aac 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -297,7 +297,7 @@ resource "aws_iam_role_policy_attachment" "node" { for_each = { for k, v in merge( { AmazonEKSWorkerNodePolicy = "${local.node_iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" - AmazonEC2ContainerRegistryReadOnly = "${local.node_iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" + AmazonEC2ContainerRegistryPullOnly = "${local.node_iam_role_policy_prefix}/AmazonEC2ContainerRegistryPullOnly" }, local.ipv4_cni_policy, local.ipv6_cni_policy diff --git a/modules/karpenter/policy.tf b/modules/karpenter/policy.tf index 16f28fa61d..ef478a08e8 100644 --- a/modules/karpenter/policy.tf +++ b/modules/karpenter/policy.tf @@ -50,6 +50,7 @@ data "aws_iam_policy_document" "controller" { "arn:${local.partition}:ec2:${local.region}:*:network-interface/*", "arn:${local.partition}:ec2:${local.region}:*:launch-template/*", "arn:${local.partition}:ec2:${local.region}:*:spot-instances-request/*", + "arn:${local.partition}:ec2:${local.region}:*:capacity-reservation/*" ] actions = [ "ec2:RunInstances", @@ -348,6 +349,12 @@ data "aws_iam_policy_document" "controller" { actions = ["iam:GetInstanceProfile"] } + statement { + sid = "AllowUnscopedInstanceProfileListAction" + resources = ["*"] + actions = ["iam:ListInstanceProfiles"] + } + statement { sid = "AllowAPIServerEndpointDiscovery" resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] From bdce0214baae49e63b140c3083ec75e3314f3d75 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 11 Sep 2025 13:28:11 +0000 Subject: [PATCH 920/962] chore(release): version 21.2.0 [skip ci] ## [21.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.5...v21.2.0) (2025-09-11) ### Features * Update Karpenter controller policy and permissions to match upstream project ([#3510](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3510)) ([131db39](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/131db3973f7eaf539c33b73014058a94ac0d0528)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b28440f1c4..0616f52cba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.5...v21.2.0) (2025-09-11) + + +### Features + +* Update Karpenter controller policy and permissions to match upstream project ([#3510](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3510)) ([131db39](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/131db3973f7eaf539c33b73014058a94ac0d0528)) + ## [21.1.5](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.4...v21.1.5) (2025-08-26) From 165d7c8c3bb15b260c23bf07fa0443c0d3accd2f Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 16 Sep 2025 08:49:00 -0500 Subject: [PATCH 921/962] feat: Support EKS Auto Mode custom node pools only creation (#3514) --- README.md | 36 ++++++++++++++++++-- examples/eks-auto-mode/README.md | 5 +-- examples/eks-auto-mode/main.tf | 19 ++++++++++- examples/eks-auto-mode/versions.tf | 2 +- examples/eks-hybrid-nodes/README.md | 6 ++-- examples/eks-hybrid-nodes/versions.tf | 2 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 6 ++-- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- main.tf | 26 +++++++------- modules/eks-managed-node-group/README.md | 4 +-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 +-- modules/fargate-profile/versions.tf | 2 +- modules/hybrid-node-role/README.md | 4 +-- modules/hybrid-node-role/versions.tf | 2 +- modules/karpenter/README.md | 4 +-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 +-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-fargate-profile/README.md | 4 +-- tests/eks-fargate-profile/versions.tf | 2 +- tests/eks-hybrid-nodes/README.md | 2 +- tests/eks-hybrid-nodes/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 +-- tests/eks-managed-node-group/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 +-- tests/self-managed-node-group/versions.tf | 2 +- variables.tf | 9 ++++- versions.tf | 2 +- 31 files changed, 113 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index ec22fdd351..84a960af20 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,35 @@ module "eks" { } ``` +### EKS Auto Mode - Custom Node Pools Only + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" + + name = "example" + kubernetes_version = "1.33" + + # Optional + endpoint_public_access = true + + # Optional: Adds the current caller identity as an administrator via cluster access entry + enable_cluster_creator_admin_permissions = true + + # Create just the IAM resources for EKS Auto Mode for use with custom node pools + create_auto_mode_iam_resources = true + + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + ### EKS Managed Node Group ```hcl @@ -318,7 +347,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 4.0 | @@ -326,7 +355,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 4.0 | @@ -393,9 +422,10 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days | `number` | `90` | no | | [cloudwatch\_log\_group\_tags](#input\_cloudwatch\_log\_group\_tags) | A map of additional tags to add to the cloudwatch log group created | `map(string)` | `{}` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | -| [compute\_config](#input\_compute\_config) | Configuration block for the cluster compute configuration |
object({
enabled = optional(bool, false)
node_pools = optional(list(string))
node_role_arn = optional(string)
})
| `null` | no | +| [compute\_config](#input\_compute\_config) | Configuration block for the cluster compute configuration |
object({
enabled = optional(bool, false)
node_pools = optional(list(string))
node_role_arn = optional(string)
})
| `{}` | no | | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | +| [create\_auto\_mode\_iam\_resources](#input\_create\_auto\_mode\_iam\_resources) | Determines whether to create/attach IAM resources for EKS Auto Mode. Useful for when using only custom node pools and not built-in EKS Auto Mode node pools | `bool` | `false` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | | [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy) | `bool` | `false` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created for the cluster | `bool` | `true` | no | diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index 204fecd0b4..3dbb1c62c6 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -25,13 +25,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | ## Modules @@ -39,6 +39,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|--------|---------| | [disabled\_eks](#module\_disabled\_eks) | ../.. | n/a | | [eks](#module\_eks) | ../.. | n/a | +| [eks\_auto\_custom\_node\_pools](#module\_eks\_auto\_custom\_node\_pools) | ../.. | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/eks-auto-mode/main.tf b/examples/eks-auto-mode/main.tf index e00b825f63..05eec9b0e4 100644 --- a/examples/eks-auto-mode/main.tf +++ b/examples/eks-auto-mode/main.tf @@ -35,7 +35,6 @@ module "eks" { name = local.name kubernetes_version = local.kubernetes_version endpoint_public_access = true - deletion_protection = true enable_cluster_creator_admin_permissions = true @@ -50,6 +49,24 @@ module "eks" { tags = local.tags } +module "eks_auto_custom_node_pools" { + source = "../.." + + name = "${local.name}-custom" + kubernetes_version = local.kubernetes_version + endpoint_public_access = true + + enable_cluster_creator_admin_permissions = true + + # Create just the IAM resources for EKS Auto Mode for use with custom node pools + create_auto_mode_iam_resources = true + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + tags = local.tags +} + module "disabled_eks" { source = "../.." diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index d2b9e1c244..f5da694d80 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -26,7 +26,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | | [helm](#requirement\_helm) | >= 3.0 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | @@ -36,8 +36,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | -| [aws.remote](#provider\_aws.remote) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | +| [aws.remote](#provider\_aws.remote) | >= 6.13 | | [helm](#provider\_helm) | >= 3.0 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index 8762781644..988eb51212 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } helm = { source = "hashicorp/helm" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 8e7e11ecf4..8465cfb6c9 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -94,15 +94,15 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | | [helm](#requirement\_helm) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | -| [aws.virginia](#provider\_aws.virginia) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | +| [aws.virginia](#provider\_aws.virginia) | >= 6.13 | | [helm](#provider\_helm) | >= 3.0 | ## Modules diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 6cd7d576fc..e8922f3f27 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/main.tf b/main.tf index 8ea8629b8d..263380b817 100644 --- a/main.tf +++ b/main.tf @@ -26,7 +26,7 @@ locals { create_outposts_local_cluster = var.outpost_config != null enable_encryption_config = var.encryption_config != null && !local.create_outposts_local_cluster - auto_mode_enabled = try(var.compute_config.enabled, false) + create_auto_mode_iam_resources = var.compute_config.enabled || var.create_auto_mode_iam_resources } ################################################################################ @@ -58,7 +58,7 @@ resource "aws_eks_cluster" "this" { } dynamic "compute_config" { - for_each = var.compute_config != null ? [var.compute_config] : [] + for_each = [var.compute_config] content { enabled = compute_config.value.enabled @@ -81,10 +81,10 @@ resource "aws_eks_cluster" "this" { content { dynamic "elastic_load_balancing" { - for_each = local.auto_mode_enabled ? [1] : [] + for_each = [var.compute_config] content { - enabled = local.auto_mode_enabled + enabled = elastic_load_balancing.value.enabled } } @@ -148,11 +148,11 @@ resource "aws_eks_cluster" "this" { } dynamic "storage_config" { - for_each = local.auto_mode_enabled ? [1] : [] + for_each = [var.compute_config] content { block_storage { - enabled = local.auto_mode_enabled + enabled = storage_config.value.enabled } } } @@ -476,7 +476,7 @@ locals { # Standard EKS cluster eks_standard_iam_role_policies = { for k, v in { AmazonEKSClusterPolicy = "${local.iam_role_policy_prefix}/AmazonEKSClusterPolicy", - } : k => v if !local.create_outposts_local_cluster && !local.auto_mode_enabled } + } : k => v if !local.create_outposts_local_cluster && !local.create_auto_mode_iam_resources } # EKS cluster with EKS auto mode enabled eks_auto_mode_iam_role_policies = { for k, v in { @@ -485,12 +485,12 @@ locals { AmazonEKSBlockStoragePolicy = "${local.iam_role_policy_prefix}/AmazonEKSBlockStoragePolicy" AmazonEKSLoadBalancingPolicy = "${local.iam_role_policy_prefix}/AmazonEKSLoadBalancingPolicy" AmazonEKSNetworkingPolicy = "${local.iam_role_policy_prefix}/AmazonEKSNetworkingPolicy" - } : k => v if !local.create_outposts_local_cluster && local.auto_mode_enabled } + } : k => v if !local.create_outposts_local_cluster && local.create_auto_mode_iam_resources } # EKS local cluster on Outposts eks_outpost_iam_role_policies = { for k, v in { AmazonEKSClusterPolicy = "${local.iam_role_policy_prefix}/AmazonEKSLocalOutpostClusterPolicy" - } : k => v if local.create_outposts_local_cluster && !local.auto_mode_enabled } + } : k => v if local.create_outposts_local_cluster && !local.create_auto_mode_iam_resources } } data "aws_iam_policy_document" "assume_role_policy" { @@ -591,7 +591,7 @@ resource "aws_iam_policy" "cluster_encryption" { } data "aws_iam_policy_document" "custom" { - count = local.create_iam_role && local.auto_mode_enabled && var.enable_auto_mode_custom_tags ? 1 : 0 + count = local.create_iam_role && local.create_auto_mode_iam_resources && var.enable_auto_mode_custom_tags ? 1 : 0 dynamic "statement" { for_each = var.enable_auto_mode_custom_tags ? [1] : [] @@ -725,7 +725,7 @@ data "aws_iam_policy_document" "custom" { } resource "aws_iam_policy" "custom" { - count = local.create_iam_role && local.auto_mode_enabled && var.enable_auto_mode_custom_tags ? 1 : 0 + count = local.create_iam_role && local.create_auto_mode_iam_resources && var.enable_auto_mode_custom_tags ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null @@ -738,7 +738,7 @@ resource "aws_iam_policy" "custom" { } resource "aws_iam_role_policy_attachment" "custom" { - count = local.create_iam_role && local.auto_mode_enabled && var.enable_auto_mode_custom_tags ? 1 : 0 + count = local.create_iam_role && local.create_auto_mode_iam_resources && var.enable_auto_mode_custom_tags ? 1 : 0 policy_arn = aws_iam_policy.custom[0].arn role = aws_iam_role.this[0].name @@ -875,7 +875,7 @@ resource "aws_eks_identity_provider_config" "this" { ################################################################################ locals { - create_node_iam_role = local.create && var.create_node_iam_role && local.auto_mode_enabled + create_node_iam_role = local.create && var.create_node_iam_role && local.create_auto_mode_iam_resources node_iam_role_name = coalesce(var.node_iam_role_name, "${var.name}-eks-auto") } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 6c923884f9..f2744c45f3 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 8981d2a4f9..4685c58b85 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index 9e7e0f4570..ce84bbd19f 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -75,13 +75,13 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | ## Modules diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 36f3412fe9..fea43a9a27 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index c14f190d47..04ad2e61c2 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index 5ff2f7f79c..28d2f06db1 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | ## Modules diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index 27875d930e..71ae477191 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -18,7 +18,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index d56b124e91..6be6e0b052 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 4b69f395b7..d53f0a633b 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index eba5c2f79b..d45c7ee922 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.9 | +| [aws](#requirement\_aws) | >= 6.13 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.9 | +| [aws](#provider\_aws) | >= 6.13 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index fe4021c7cf..8f71cb6086 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } } } diff --git a/variables.tf b/variables.tf index 9647017980..0b155d0786 100644 --- a/variables.tf +++ b/variables.tf @@ -69,7 +69,8 @@ variable "compute_config" { node_pools = optional(list(string)) node_role_arn = optional(string) }) - default = null + default = {} + nullable = false } variable "upgrade_policy" { @@ -611,6 +612,12 @@ variable "enable_auto_mode_custom_tags" { default = true } +variable "create_auto_mode_iam_resources" { + description = "Determines whether to create/attach IAM resources for EKS Auto Mode. Useful for when using only custom node pools and not built-in EKS Auto Mode node pools" + type = bool + default = false +} + ################################################################################ # EKS Addons ################################################################################ diff --git a/versions.tf b/versions.tf index 6c8090303a..d6ab6ed94e 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.9" + version = ">= 6.13" } tls = { source = "hashicorp/tls" From 0b29f2bcb2532154699f68e52e0c7420d6355a8e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 16 Sep 2025 13:49:34 +0000 Subject: [PATCH 922/962] chore(release): version 21.3.0 [skip ci] ## [21.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.2.0...v21.3.0) (2025-09-16) ### Features * Support EKS Auto Mode custom node pools only creation ([#3514](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3514)) ([165d7c8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/165d7c8c3bb15b260c23bf07fa0443c0d3accd2f)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0616f52cba..bc9ca52083 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.2.0...v21.3.0) (2025-09-16) + + +### Features + +* Support EKS Auto Mode custom node pools only creation ([#3514](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3514)) ([165d7c8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/165d7c8c3bb15b260c23bf07fa0443c0d3accd2f)) + ## [21.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.1.5...v21.2.0) (2025-09-11) From c8bb152839c411247321194531eadbd7dcdeced4 Mon Sep 17 00:00:00 2001 From: Erez Zarum Date: Tue, 16 Sep 2025 20:14:18 +0300 Subject: [PATCH 923/962] fix: Sync Karpenter IAM permissions with upstream (#3517) Sync Karpenter IAM permissions with upstream Signed-off-by: Erez Zarum --- modules/karpenter/policy.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/karpenter/policy.tf b/modules/karpenter/policy.tf index ef478a08e8..1b3df556ea 100644 --- a/modules/karpenter/policy.tf +++ b/modules/karpenter/policy.tf @@ -181,6 +181,7 @@ data "aws_iam_policy_document" "controller" { sid = "AllowRegionalReadActions" resources = ["*"] actions = [ + "ec2:DescribeCapacityReservations", "ec2:DescribeAvailabilityZones", "ec2:DescribeImages", "ec2:DescribeInstances", From de2aa10f25c7f2d2ab1264f6451f7cbf57f784c4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 16 Sep 2025 17:14:54 +0000 Subject: [PATCH 924/962] chore(release): version 21.3.1 [skip ci] ## [21.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.3.0...v21.3.1) (2025-09-16) ### Bug Fixes * Sync Karpenter IAM permissions with upstream ([#3517](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3517)) ([c8bb152](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c8bb152839c411247321194531eadbd7dcdeced4)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc9ca52083..1a4ab931eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.3.0...v21.3.1) (2025-09-16) + + +### Bug Fixes + +* Sync Karpenter IAM permissions with upstream ([#3517](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3517)) ([c8bb152](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c8bb152839c411247321194531eadbd7dcdeced4)) + ## [21.3.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.2.0...v21.3.0) (2025-09-16) From bf8c86f3fa66fdbd372bfec229656cda7a8e6036 Mon Sep 17 00:00:00 2001 From: moko-poi Date: Mon, 6 Oct 2025 23:43:03 +0900 Subject: [PATCH 925/962] docs: Update Karpenter example to use `region` argument in place of aliased AWS provider (#3538) * feat(karpenter): support AWS Provider v6 region attribute * docs(karpenter): remove unused aws.virginia provider reference --- examples/karpenter/README.md | 1 - examples/karpenter/main.tf | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 8465cfb6c9..f7400b7e26 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -102,7 +102,6 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 6.13 | -| [aws.virginia](#provider\_aws.virginia) | >= 6.13 | | [helm](#provider\_helm) | >= 3.0 | ## Modules diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 6f652327c3..c61b4b9920 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -2,11 +2,6 @@ provider "aws" { region = local.region } -provider "aws" { - region = "us-east-1" - alias = "virginia" -} - provider "helm" { kubernetes = { host = module.eks.cluster_endpoint @@ -30,7 +25,7 @@ data "aws_availability_zones" "available" { } data "aws_ecrpublic_authorization_token" "token" { - provider = aws.virginia + region = "us-east-1" } locals { From f5f6dae50737137d8709b5fe2f4129a1251eacca Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 6 Oct 2025 11:49:27 -0500 Subject: [PATCH 926/962] fix: Incorporate AWS provider `v6.15` corrections for EKS Auto Mode to support enabling/disabling EKS Auto Mode without affecting non-Auto Mode users (#3526) * fix: Raise min supported version of AWS provider for EKS Auto Mode corrections * docs: Add note on encryption config settings * fix: Revert forcing a value for all EKS Auto Mode fields now that provider handles this --- .pre-commit-config.yaml | 2 +- README.md | 20 +++++++++++++++++--- docs/UPGRADE-21.0.md | 1 + examples/eks-auto-mode/README.md | 4 ++-- examples/eks-auto-mode/main.tf | 3 +++ examples/eks-auto-mode/versions.tf | 2 +- examples/eks-hybrid-nodes/README.md | 6 +++--- examples/eks-hybrid-nodes/versions.tf | 2 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 4 ++-- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- main.tf | 8 ++++---- modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/hybrid-node-role/README.md | 4 ++-- modules/hybrid-node-role/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-fargate-profile/README.md | 4 ++-- tests/eks-fargate-profile/versions.tf | 2 +- tests/eks-hybrid-nodes/README.md | 2 +- tests/eks-hybrid-nodes/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- variables.tf | 3 +-- versions.tf | 2 +- 33 files changed, 66 insertions(+), 49 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7cf0036a18..5198bb3f6b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.100.0 + rev: v1.101.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index 84a960af20..f596586f76 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,17 @@ Please note that we strive to provide a comprehensive suite of documentation for ### EKS Auto Mode +> [!CAUTION] +> Due to the current EKS Auto Mode API, to disable EKS Auto Mode you will have to explicity set: +> +>```hcl +>compute_config = { +> enabled = false +> } +>``` +> +> If you try to disable by simply removing the `compute_config` block, this will fail to disble EKS Auto Mode. Only after applying with `enabled = false` can you then remove the `compute_config` block from your configurations. + ```hcl module "eks" { source = "terraform-aws-modules/eks/aws" @@ -75,6 +86,9 @@ module "eks" { # Create just the IAM resources for EKS Auto Mode for use with custom node pools create_auto_mode_iam_resources = true + compute_config = { + enabled = true + } vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] @@ -347,7 +361,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 4.0 | @@ -355,7 +369,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 4.0 | @@ -422,7 +436,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 90 days | `number` | `90` | no | | [cloudwatch\_log\_group\_tags](#input\_cloudwatch\_log\_group\_tags) | A map of additional tags to add to the cloudwatch log group created | `map(string)` | `{}` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | -| [compute\_config](#input\_compute\_config) | Configuration block for the cluster compute configuration |
object({
enabled = optional(bool, false)
node_pools = optional(list(string))
node_role_arn = optional(string)
})
| `{}` | no | +| [compute\_config](#input\_compute\_config) | Configuration block for the cluster compute configuration |
object({
enabled = optional(bool, false)
node_pools = optional(list(string))
node_role_arn = optional(string)
})
| `null` | no | | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_auto\_mode\_iam\_resources](#input\_create\_auto\_mode\_iam\_resources) | Determines whether to create/attach IAM resources for EKS Auto Mode. Useful for when using only custom node pools and not built-in EKS Auto Mode node pools | `bool` | `false` | no | diff --git a/docs/UPGRADE-21.0.md b/docs/UPGRADE-21.0.md index 695c8c80f2..9d6500b1d4 100644 --- a/docs/UPGRADE-21.0.md +++ b/docs/UPGRADE-21.0.md @@ -32,6 +32,7 @@ If you find a bug, please open an issue with supporting configuration to reprodu - `addons.most_recent` is now set to `true` by default (was `false`). - `cluster_identity_providers.issuer_url` is now required to be set by users; the prior incorrect default has been removed. See https://github.com/terraform-aws-modules/terraform-aws-eks/pull/3055 and https://github.com/kubernetes/kubernetes/pull/123561 for more details. - The OIDC issuer URL for IAM roles for service accounts (IRSA) has been changed to use the new dual stack`oidc-eks` endpoint instead of `oidc.eks`. This is to align with https://github.com/aws/containers-roadmap/issues/2038#issuecomment-2278450601 +- With the changes to the variable type definition for `encryption_config` (formerly `cluster_encryption_config`), if you wish to disable secret encryption with a custom KMS key you should set `encryption_config = null` (In `v20.x`, you would previously have set `encryption_config = {}` to achieve the same outcome). Secret encryption can no longer be disabled - it is either enabled by default with the AWS managed key (`encryption_config = null`), or with a custom KMS key ( either leaving as is by not specifying or passing your own custom key ARN). EKS now encrypts secrets at rest by default docs.aws.amazon.com/eks/latest/userguide/envelope-encryption.html and the default secret encryption w/ custom KMS key creation/usage by default was made years prior starting in version `v19.0` of this module. Removing this default behavior will be evaluated at the next breaking change given that secrets are now automatically encrypted at rest by AWS. ## Additional changes diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index 3dbb1c62c6..57cdfa7d4b 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -25,13 +25,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | ## Modules diff --git a/examples/eks-auto-mode/main.tf b/examples/eks-auto-mode/main.tf index 05eec9b0e4..02dc255a24 100644 --- a/examples/eks-auto-mode/main.tf +++ b/examples/eks-auto-mode/main.tf @@ -60,6 +60,9 @@ module "eks_auto_custom_node_pools" { # Create just the IAM resources for EKS Auto Mode for use with custom node pools create_auto_mode_iam_resources = true + compute_config = { + enabled = true + } vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index f5da694d80..e4afac8bec 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -26,7 +26,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | | [helm](#requirement\_helm) | >= 3.0 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | @@ -36,8 +36,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | -| [aws.remote](#provider\_aws.remote) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | +| [aws.remote](#provider\_aws.remote) | >= 6.15 | | [helm](#provider\_helm) | >= 3.0 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index 988eb51212..74ba3338c5 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } helm = { source = "hashicorp/helm" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index f7400b7e26..81863c0248 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -94,14 +94,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | | [helm](#requirement\_helm) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | | [helm](#provider\_helm) | >= 3.0 | ## Modules diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index e8922f3f27..0c6561a848 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/main.tf b/main.tf index 263380b817..5d260d4dae 100644 --- a/main.tf +++ b/main.tf @@ -26,7 +26,7 @@ locals { create_outposts_local_cluster = var.outpost_config != null enable_encryption_config = var.encryption_config != null && !local.create_outposts_local_cluster - create_auto_mode_iam_resources = var.compute_config.enabled || var.create_auto_mode_iam_resources + create_auto_mode_iam_resources = try(var.compute_config.enabled, false) == true || var.create_auto_mode_iam_resources } ################################################################################ @@ -58,7 +58,7 @@ resource "aws_eks_cluster" "this" { } dynamic "compute_config" { - for_each = [var.compute_config] + for_each = var.compute_config != null ? [var.compute_config] : [] content { enabled = compute_config.value.enabled @@ -81,7 +81,7 @@ resource "aws_eks_cluster" "this" { content { dynamic "elastic_load_balancing" { - for_each = [var.compute_config] + for_each = var.compute_config != null ? [var.compute_config] : [] content { enabled = elastic_load_balancing.value.enabled @@ -148,7 +148,7 @@ resource "aws_eks_cluster" "this" { } dynamic "storage_config" { - for_each = [var.compute_config] + for_each = var.compute_config != null ? [var.compute_config] : [] content { block_storage { diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index f2744c45f3..1ca9c95095 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 4685c58b85..6c260145ff 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index ce84bbd19f..45285c033f 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -75,13 +75,13 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | ## Modules diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index fea43a9a27..df1b123479 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 04ad2e61c2..fb04648cea 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index 28d2f06db1..600f83d5af 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | ## Modules diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index 71ae477191..6c6cd65252 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -18,7 +18,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index 6be6e0b052..782544463d 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index d53f0a633b..9b415f283b 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index d45c7ee922..df4457cef4 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.13 | +| [aws](#requirement\_aws) | >= 6.15 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.13 | +| [aws](#provider\_aws) | >= 6.15 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index 8f71cb6086..e1ac31cae0 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } } } diff --git a/variables.tf b/variables.tf index 0b155d0786..842d473499 100644 --- a/variables.tf +++ b/variables.tf @@ -69,8 +69,7 @@ variable "compute_config" { node_pools = optional(list(string)) node_role_arn = optional(string) }) - default = {} - nullable = false + default = null } variable "upgrade_policy" { diff --git a/versions.tf b/versions.tf index d6ab6ed94e..0ebaec8d6b 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.13" + version = ">= 6.15" } tls = { source = "hashicorp/tls" From 74824da9c4fe9dd0b405db70881a1158fa1af216 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 6 Oct 2025 16:49:54 +0000 Subject: [PATCH 927/962] chore(release): version 21.3.2 [skip ci] ## [21.3.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.3.1...v21.3.2) (2025-10-06) ### Bug Fixes * Incorporate AWS provider `v6.15` corrections for EKS Auto Mode to support enabling/disabling EKS Auto Mode without affecting non-Auto Mode users ([#3526](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3526)) ([f5f6dae](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f5f6dae50737137d8709b5fe2f4129a1251eacca)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a4ab931eb..4ff19fed49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.3.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.3.1...v21.3.2) (2025-10-06) + + +### Bug Fixes + +* Incorporate AWS provider `v6.15` corrections for EKS Auto Mode to support enabling/disabling EKS Auto Mode without affecting non-Auto Mode users ([#3526](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3526)) ([f5f6dae](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/f5f6dae50737137d8709b5fe2f4129a1251eacca)) + ## [21.3.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.3.0...v21.3.1) (2025-09-16) From fd490ea897117f3c9346c600cceece6b3fead7e7 Mon Sep 17 00:00:00 2001 From: Rahul Nimbalkar <36779043+ranimbal@users.noreply.github.com> Date: Tue, 14 Oct 2025 09:18:12 -0400 Subject: [PATCH 928/962] feat: Allow setting KMS key rotation period (#3546) feat: make kms key rotation period configurable Co-authored-by: Rahul Nimbalkar --- README.md | 1 + main.tf | 1 + variables.tf | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/README.md b/README.md index f596586f76..08b05dfff4 100644 --- a/README.md +++ b/README.md @@ -486,6 +486,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [kms\_key\_enable\_default\_policy](#input\_kms\_key\_enable\_default\_policy) | Specifies whether to enable the default key policy | `bool` | `true` | no | | [kms\_key\_override\_policy\_documents](#input\_kms\_key\_override\_policy\_documents) | List of IAM policy documents that are merged together into the exported document. In merging, statements with non-blank `sid`s will override statements with the same `sid` | `list(string)` | `[]` | no | | [kms\_key\_owners](#input\_kms\_key\_owners) | A list of IAM ARNs for those who will have full key permissions (`kms:*`) | `list(string)` | `[]` | no | +| [kms\_key\_rotation\_period\_in\_days](#input\_kms\_key\_rotation\_period\_in\_days) | Custom period of time between each key rotation date. If you specify a value, it must be between `90` and `2560`, inclusive. If you do not specify a value, it defaults to `365` | `number` | `null` | no | | [kms\_key\_service\_users](#input\_kms\_key\_service\_users) | A list of IAM ARNs for [key service users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-service-integration) | `list(string)` | `[]` | no | | [kms\_key\_source\_policy\_documents](#input\_kms\_key\_source\_policy\_documents) | List of IAM policy documents that are merged together into the exported document. Statements must have unique `sid`s | `list(string)` | `[]` | no | | [kms\_key\_users](#input\_kms\_key\_users) | A list of IAM ARNs for [key users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-users) | `list(string)` | `[]` | no | diff --git a/main.tf b/main.tf index 5d260d4dae..7a6ddbd2e4 100644 --- a/main.tf +++ b/main.tf @@ -340,6 +340,7 @@ module "kms" { key_usage = "ENCRYPT_DECRYPT" deletion_window_in_days = var.kms_key_deletion_window_in_days enable_key_rotation = var.enable_kms_key_rotation + rotation_period_in_days = var.kms_key_rotation_period_in_days # Policy enable_default_policy = var.kms_key_enable_default_policy diff --git a/variables.tf b/variables.tf index 842d473499..6f3497eacf 100644 --- a/variables.tf +++ b/variables.tf @@ -263,6 +263,12 @@ variable "enable_kms_key_rotation" { default = true } +variable "kms_key_rotation_period_in_days" { + description = "Custom period of time between each key rotation date. If you specify a value, it must be between `90` and `2560`, inclusive. If you do not specify a value, it defaults to `365`" + type = number + default = null +} + variable "kms_key_enable_default_policy" { description = "Specifies whether to enable the default key policy" type = bool From e4bc4d8fd59aac359dca88a5a3fbe03907fc8930 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 14 Oct 2025 13:18:40 +0000 Subject: [PATCH 929/962] chore(release): version 21.4.0 [skip ci] ## [21.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.3.2...v21.4.0) (2025-10-14) ### Features * Allow setting KMS key rotation period ([#3546](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3546)) ([fd490ea](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/fd490ea897117f3c9346c600cceece6b3fead7e7)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ff19fed49..5bb6fc2769 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.3.2...v21.4.0) (2025-10-14) + + +### Features + +* Allow setting KMS key rotation period ([#3546](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3546)) ([fd490ea](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/fd490ea897117f3c9346c600cceece6b3fead7e7)) + ## [21.3.2](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.3.1...v21.3.2) (2025-10-06) From 67557e8fe866dafd318a9c1d79b08bd9615a839b Mon Sep 17 00:00:00 2001 From: Evan Date: Mon, 20 Oct 2025 16:18:50 -0400 Subject: [PATCH 930/962] feat: Allow for additional policy statements on sqs queue policy (#3543) * feat: allow for additional policy statements on sqs queue policy * fix: Update variable type and run `pre-commit` * fix: Update variable name and description --------- Co-authored-by: Evan Shoshan Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 2 +- modules/karpenter/README.md | 3 ++- modules/karpenter/main.tf | 42 ++++++++++++++++++++++++++++++++++ modules/karpenter/variables.tf | 28 ++++++++++++++++++++++- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5198bb3f6b..260cc8c757 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.101.0 + rev: v1.103.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index df1b123479..e31bb3073d 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -142,7 +142,7 @@ No modules. | [iam\_policy\_description](#input\_iam\_policy\_description) | IAM policy description | `string` | `"Karpenter controller IAM policy"` | no | | [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy | `string` | `"KarpenterController"` | no | | [iam\_policy\_path](#input\_iam\_policy\_path) | Path of the IAM policy | `string` | `"/"` | no | -| [iam\_policy\_statements](#input\_iam\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed |
list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
}))
| `null` | no | +| [iam\_policy\_statements](#input\_iam\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed |
list(object({ # TODO - change to `map(object({...}))` in next major version
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
}))
| `null` | no | | [iam\_policy\_use\_name\_prefix](#input\_iam\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`iam_policy_name`) is used as a prefix | `bool` | `true` | no | | [iam\_role\_description](#input\_iam\_role\_description) | IAM role description | `string` | `"Karpenter controller IAM role"` | no | | [iam\_role\_max\_session\_duration](#input\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | @@ -169,6 +169,7 @@ No modules. | [queue\_kms\_master\_key\_id](#input\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no | | [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | | [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no | +| [queue\_policy\_statements](#input\_queue\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific SQS queue policy permissions as needed |
map(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
}))
| `null` | no | | [region](#input\_region) | Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration | `string` | `null` | no | | [rule\_name\_prefix](#input\_rule\_name\_prefix) | Prefix used for all event bridge rules | `string` | `"Karpenter"` | no | | [service\_account](#input\_service\_account) | Service account to associate with the Karpenter Pod Identity | `string` | `"karpenter"` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 227a2f6aac..53e961244c 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -145,6 +145,7 @@ data "aws_iam_policy_document" "queue" { ] } } + statement { sid = "DenyHTTP" effect = "Deny" @@ -166,6 +167,47 @@ data "aws_iam_policy_document" "queue" { ] } } + + dynamic "statement" { + for_each = var.queue_policy_statements != null ? var.queue_policy_statements : {} + + content { + sid = try(coalesce(statement.value.sid, statement.key)) + actions = statement.value.actions + not_actions = statement.value.not_actions + effect = statement.value.effect + resources = statement.value.resources + not_resources = statement.value.not_resources + + dynamic "principals" { + for_each = statement.value.principals != null ? statement.value.principals : [] + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = statement.value.not_principals != null ? statement.value.not_principals : [] + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = statement.value.condition != null ? statement.value.condition : [] + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } } resource "aws_sqs_queue_policy" "this" { diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index f0725f6a58..6d3d042516 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -112,7 +112,7 @@ variable "iam_role_source_assume_policy_documents" { variable "iam_policy_statements" { description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" - type = list(object({ + type = list(object({ # TODO - change to `map(object({...}))` in next major version sid = optional(string) actions = optional(list(string)) not_actions = optional(list(string)) @@ -204,6 +204,32 @@ variable "queue_kms_data_key_reuse_period_seconds" { default = null } +variable "queue_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific SQS queue policy permissions as needed" + type = map(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + })) + default = null +} + ################################################################################ # Node IAM Role ################################################################################ From 4c77bbc1198e85655d5324f18636ad16a3ca9adf Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 20 Oct 2025 20:19:45 +0000 Subject: [PATCH 931/962] chore(release): version 21.5.0 [skip ci] ## [21.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.4.0...v21.5.0) (2025-10-20) ### Features * Allow for additional policy statements on sqs queue policy ([#3543](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3543)) ([67557e8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/67557e8fe866dafd318a9c1d79b08bd9615a839b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bb6fc2769..9f37d154d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.4.0...v21.5.0) (2025-10-20) + + +### Features + +* Allow for additional policy statements on sqs queue policy ([#3543](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3543)) ([67557e8](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/67557e8fe866dafd318a9c1d79b08bd9615a839b)) + ## [21.4.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.3.2...v21.4.0) (2025-10-14) From 0b0ca6601923e8542f2f692994d5cb0671823c46 Mon Sep 17 00:00:00 2001 From: Fred Myerscough <79988537+myerscf@users.noreply.github.com> Date: Mon, 20 Oct 2025 21:39:27 +0100 Subject: [PATCH 932/962] feat: Use `aws_service_principal` data source for deriving IAM service prinicpals (#3539) * fix: removed use of dns_suffix use the service prinipal name in policies Signed-off-by: Fred Myerscough * fix: added missing data. accessor Signed-off-by: Fred Myerscough * refactor: updates to address pr comments Signed-off-by: Fred Myerscough * refactor: updates to address pr comments Signed-off-by: Fred Myerscough * fix: Remove unused local var, run pre-commit --------- Signed-off-by: Fred Myerscough Co-authored-by: Bryant Biggs --- modules/karpenter/README.md | 1 + modules/karpenter/main.tf | 18 +++++++++++++----- modules/karpenter/policy.tf | 2 +- outputs.tf | 4 ---- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index e31bb3073d..ad8dfcd979 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -123,6 +123,7 @@ No modules. | [aws_iam_policy_document.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | +| [aws_service_principal.ec2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/service_principal) | data source | ## Inputs diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 53e961244c..4d3cc08ed1 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -3,18 +3,26 @@ data "aws_region" "current" { region = var.region } + data "aws_partition" "current" { count = var.create ? 1 : 0 } + data "aws_caller_identity" "current" { count = var.create ? 1 : 0 } +data "aws_service_principal" "ec2" { + count = var.create ? 1 : 0 + + service_name = "ec2" +} + locals { - account_id = try(data.aws_caller_identity.current[0].account_id, "") - dns_suffix = try(data.aws_partition.current[0].dns_suffix, "") - partition = try(data.aws_partition.current[0].partition, "") - region = try(data.aws_region.current[0].region, "") + account_id = try(data.aws_caller_identity.current[0].account_id, "") + ec2_sp_name = try(data.aws_service_principal.ec2[0].name, "") + partition = try(data.aws_partition.current[0].partition, "") + region = try(data.aws_region.current[0].region, "") } ################################################################################ @@ -313,7 +321,7 @@ data "aws_iam_policy_document" "node_assume_role" { principals { type = "Service" - identifiers = ["ec2.${local.dns_suffix}"] + identifiers = [local.ec2_sp_name] } } } diff --git a/modules/karpenter/policy.tf b/modules/karpenter/policy.tf index 1b3df556ea..34937f36eb 100644 --- a/modules/karpenter/policy.tf +++ b/modules/karpenter/policy.tf @@ -234,7 +234,7 @@ data "aws_iam_policy_document" "controller" { condition { test = "StringEquals" variable = "iam:PassedToService" - values = distinct(["ec2.${local.dns_suffix}", "ec2.amazonaws.com"]) + values = distinct([local.ec2_sp_name, "ec2.amazonaws.com"]) } } diff --git a/outputs.tf b/outputs.tf index 4663b83639..ce5756cef5 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,7 +1,3 @@ -locals { - -} - ################################################################################ # Cluster ################################################################################ From b6ffccf137fad60e09a9358f75d356f7f3e97d1b Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 20 Oct 2025 20:39:58 +0000 Subject: [PATCH 933/962] chore(release): version 21.6.0 [skip ci] ## [21.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.5.0...v21.6.0) (2025-10-20) ### Features * Use `aws_service_principal` data source for deriving IAM service prinicpals ([#3539](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3539)) ([0b0ca66](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0b0ca6601923e8542f2f692994d5cb0671823c46)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f37d154d7..93b38d36bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [21.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.5.0...v21.6.0) (2025-10-20) + + +### Features + +* Use `aws_service_principal` data source for deriving IAM service prinicpals ([#3539](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3539)) ([0b0ca66](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0b0ca6601923e8542f2f692994d5cb0671823c46)) + ## [21.5.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.4.0...v21.5.0) (2025-10-20) From e4e25b1c4883698a209b23155005fb2a5257c265 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 21 Oct 2025 03:59:10 -0500 Subject: [PATCH 934/962] fix: Update CI workflow versions to latest (#3554) --- .github/workflows/pr-title.yml | 2 +- .github/workflows/pre-commit.yml | 111 +++++++++++++++++---- .github/workflows/release.yml | 22 ++-- .github/workflows/stale-actions.yaml | 2 +- .gitignore | 23 ++--- {.github => docs}/CHANGELOG.pre-v11.0.0.md | 0 examples/README.md | 5 +- 7 files changed, 121 insertions(+), 44 deletions(-) rename {.github => docs}/CHANGELOG.pre-v11.0.0.md (100%) diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 1e50760ee7..6419f3aa90 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -14,7 +14,7 @@ jobs: steps: # Please look up the latest version from # https://github.com/amannn/action-semantic-pull-request/releases - - uses: amannn/action-semantic-pull-request@v5.5.3 + - uses: amannn/action-semantic-pull-request@v6.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index a19ff831f9..057b9c42e2 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -7,8 +7,8 @@ on: - master env: - TERRAFORM_DOCS_VERSION: v0.19.0 - TFLINT_VERSION: v0.53.0 + TERRAFORM_DOCS_VERSION: v0.20.0 + TFLINT_VERSION: v0.59.1 jobs: collectInputs: @@ -18,11 +18,11 @@ jobs: directories: ${{ steps.dirs.outputs.directories }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Get root directories id: dirs - uses: clowdhaus/terraform-composite-actions/directories@v1.9.0 + uses: clowdhaus/terraform-composite-actions/directories@v1.14.0 preCommitMinVersions: name: Min TF pre-commit @@ -32,27 +32,49 @@ jobs: matrix: directory: ${{ fromJson(needs.collectInputs.outputs.directories) }} steps: + - name: Install rmz + uses: jaxxstorm/action-install-gh-release@v2.1.0 + with: + repo: SUPERCILEX/fuc + asset-name: x86_64-unknown-linux-gnu-rmz + rename-to: rmz + chmod: 0755 + extension-matching: disable + # https://github.com/orgs/community/discussions/25678#discussioncomment-5242449 - - name: Delete huge unnecessary tools folder + - name: Delete unnecessary files run: | - rm -rf /opt/hostedtoolcache/CodeQL - rm -rf /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk - rm -rf /opt/hostedtoolcache/Ruby - rm -rf /opt/hostedtoolcache/go + formatByteCount() { echo $(numfmt --to=iec-i --suffix=B --padding=7 $1'000'); } + getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); } + + BEFORE=$(getAvailableSpace) + + ln -s /opt/hostedtoolcache/SUPERCILEX/x86_64-unknown-linux-gnu-rmz/latest/linux-x64/rmz /usr/local/bin/rmz + rmz -f /opt/hostedtoolcache/CodeQL & + rmz -f /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk & + rmz -f /opt/hostedtoolcache/PyPy & + rmz -f /opt/hostedtoolcache/Ruby & + rmz -f /opt/hostedtoolcache/go & + + wait + + AFTER=$(getAvailableSpace) + SAVED=$((AFTER-BEFORE)) + echo "=> Saved $(formatByteCount $SAVED)" - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.3.1 + uses: clowdhaus/terraform-min-max@v2.1.0 with: directory: ${{ matrix.directory }} - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory != '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.11.1 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.14.0 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} tflint-version: ${{ env.TFLINT_VERSION }} @@ -61,7 +83,7 @@ jobs: - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory == '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.11.1 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.14.0 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} tflint-version: ${{ env.TFLINT_VERSION }} @@ -72,26 +94,73 @@ jobs: runs-on: ubuntu-latest needs: collectInputs steps: + - name: Install rmz + uses: jaxxstorm/action-install-gh-release@v2.1.0 + with: + repo: SUPERCILEX/fuc + asset-name: x86_64-unknown-linux-gnu-rmz + rename-to: rmz + chmod: 0755 + extension-matching: disable + # https://github.com/orgs/community/discussions/25678#discussioncomment-5242449 - - name: Delete huge unnecessary tools folder + - name: Delete unnecessary files run: | - rm -rf /opt/hostedtoolcache/CodeQL - rm -rf /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk - rm -rf /opt/hostedtoolcache/Ruby - rm -rf /opt/hostedtoolcache/go + formatByteCount() { echo $(numfmt --to=iec-i --suffix=B --padding=7 $1'000'); } + getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); } + + BEFORE=$(getAvailableSpace) + + ln -s /opt/hostedtoolcache/SUPERCILEX/x86_64-unknown-linux-gnu-rmz/latest/linux-x64/rmz /usr/local/bin/rmz + rmz -f /opt/hostedtoolcache/CodeQL & + rmz -f /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk & + rmz -f /opt/hostedtoolcache/PyPy & + rmz -f /opt/hostedtoolcache/Ruby & + rmz -f /opt/hostedtoolcache/go & + sudo rmz -f /usr/local/lib/android & + + if [[ ${{ github.repository }} == terraform-aws-modules/terraform-aws-security-group ]]; then + sudo rmz -f /usr/share/dotnet & + sudo rmz -f /usr/local/.ghcup & + sudo apt-get -qq remove -y 'azure-.*' + sudo apt-get -qq remove -y 'cpp-.*' + sudo apt-get -qq remove -y 'dotnet-runtime-.*' + sudo apt-get -qq remove -y 'google-.*' + sudo apt-get -qq remove -y 'libclang-.*' + sudo apt-get -qq remove -y 'libllvm.*' + sudo apt-get -qq remove -y 'llvm-.*' + sudo apt-get -qq remove -y 'mysql-.*' + sudo apt-get -qq remove -y 'postgresql-.*' + sudo apt-get -qq remove -y 'php.*' + sudo apt-get -qq remove -y 'temurin-.*' + sudo apt-get -qq remove -y kubectl firefox mono-devel + sudo apt-get -qq autoremove -y + sudo apt-get -qq clean + fi + + wait + + AFTER=$(getAvailableSpace) + SAVED=$((AFTER-BEFORE)) + echo "=> Saved $(formatByteCount $SAVED)" - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: ${{ github.event.pull_request.head.ref }} repository: ${{github.event.pull_request.head.repo.full_name}} - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.3.1 + uses: clowdhaus/terraform-min-max@v2.1.0 + + - name: Hide template dir + # Special to this repo, we don't want to check this dir + if: ${{ github.repository == 'terraform-aws-modules/terraform-aws-security-group' }} + run: rm -rf modules/_templates - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.11.1 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.14.0 with: terraform-version: ${{ steps.minMax.outputs.maxVersion }} tflint-version: ${{ env.TFLINT_VERSION }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a9422614e..e739b790db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,18 +20,26 @@ jobs: if: github.repository_owner == 'terraform-aws-modules' steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: persist-credentials: false fetch-depth: 0 + - name: Set correct Node.js version + uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Install dependencies + run: | + npm install \ + @semantic-release/changelog@6.0.3 \ + @semantic-release/git@10.0.1 \ + conventional-changelog-conventionalcommits@9.1.0 + - name: Release - uses: cycjimmy/semantic-release-action@v4 + uses: cycjimmy/semantic-release-action@v5 with: - semantic_version: 23.0.2 - extra_plugins: | - @semantic-release/changelog@6.0.3 - @semantic-release/git@10.0.1 - conventional-changelog-conventionalcommits@7.0.2 + semantic_version: 25.0.0 env: GITHUB_TOKEN: ${{ secrets.SEMANTIC_RELEASE_TOKEN }} diff --git a/.github/workflows/stale-actions.yaml b/.github/workflows/stale-actions.yaml index 6ccd0ed856..3e826dcfa3 100644 --- a/.github/workflows/stale-actions.yaml +++ b/.github/workflows/stale-actions.yaml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Staling issues and PR's diff --git a/.gitignore b/.gitignore index c78b7d1e2c..fd39819e6f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,3 @@ -eks-admin-cluster-role-binding.yaml -eks-admin-service-account.yaml -config-map-aws-auth*.yaml -kubeconfig_* -.idea - -################################################################# -# Default .gitignore content for all terraform-aws-modules below -################################################################# - -.DS_Store - # Local .terraform directories **/.terraform/* @@ -19,7 +7,6 @@ kubeconfig_* # .tfstate files *.tfstate *.tfstate.* -*.tfplan # Crash log files crash.log @@ -40,3 +27,13 @@ override.tf.json # Ignore CLI configuration files .terraformrc terraform.rc + +# Lambda build artifacts +builds/ +__pycache__/ +*.zip +.tox + +# Local editors/macos files +.DS_Store +.idea diff --git a/.github/CHANGELOG.pre-v11.0.0.md b/docs/CHANGELOG.pre-v11.0.0.md similarity index 100% rename from .github/CHANGELOG.pre-v11.0.0.md rename to docs/CHANGELOG.pre-v11.0.0.md diff --git a/examples/README.md b/examples/README.md index 8d54b3db38..f417c0adc0 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,8 @@ # Examples -The examples provided demonstrate different cluster configurations that users can create with the modules provided. +Please note - the examples provided serve two primary means: + +1. Show users working examples of the various ways in which the module can be configured and features supported +2. A means of testing/validating module changes Please do not mistake the examples provided as "best practices". It is up to users to consult the AWS service documentation for best practices, usage recommendations, etc. From 96dbaa033bdc9e0a23421eb77c87b4533c44a225 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 21 Oct 2025 08:59:44 +0000 Subject: [PATCH 935/962] chore(release): version 21.6.1 [skip ci] ## [21.6.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.6.0...v21.6.1) (2025-10-21) ### Bug Fixes * Update CI workflow versions to latest ([#3554](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3554)) ([e4e25b1](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e4e25b1c4883698a209b23155005fb2a5257c265)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b38d36bc..ca7776e491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.6.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.6.0...v21.6.1) (2025-10-21) + +### Bug Fixes + +* Update CI workflow versions to latest ([#3554](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3554)) ([e4e25b1](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/e4e25b1c4883698a209b23155005fb2a5257c265)) + ## [21.6.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.5.0...v21.6.0) (2025-10-20) From de8c550d5d10017eba4b2c0b492360511fe0c34b Mon Sep 17 00:00:00 2001 From: Peter Winter Date: Mon, 27 Oct 2025 20:40:06 +0100 Subject: [PATCH 936/962] feat: Add recommended security group rule for port `10251` to match EKS addon for `metrics-server` (#3562) * add current metrics-server default port 10251 to the recommended security group rules, add TODO note to remove the metrics-server legacy port 4443 on the next breaking change * Update node_groups.tf Co-authored-by: Bryant Biggs --------- Co-authored-by: Bryant Biggs --- node_groups.tf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/node_groups.tf b/node_groups.tf index 5a1655613e..f43a3e325d 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -117,7 +117,7 @@ locals { type = "ingress" self = true } - # metrics-server + # metrics-server, legacy port - TODO: remove this on the next breaking change at v22 ingress_cluster_4443_webhook = { description = "Cluster API to node 4443/tcp webhook" protocol = "tcp" @@ -126,6 +126,15 @@ locals { type = "ingress" source_cluster_security_group = true } + # metrics-server, current EKS default port + ingress_cluster_10251_webhook = { + description = "Cluster API to node 10251/tcp webhook" + protocol = "tcp" + from_port = 10251 + to_port = 10251 + type = "ingress" + source_cluster_security_group = true + } # prometheus-adapter ingress_cluster_6443_webhook = { description = "Cluster API to node 6443/tcp webhook" From 0778bbbc10063bf5672efc4ab3133f81e6686ffb Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 27 Oct 2025 19:40:42 +0000 Subject: [PATCH 937/962] chore(release): version 21.7.0 [skip ci] ## [21.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.6.1...v21.7.0) (2025-10-27) ### Features * Add recommended security group rule for port `10251` to match EKS addon for `metrics-server` ([#3562](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3562)) ([de8c550](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/de8c550d5d10017eba4b2c0b492360511fe0c34b)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca7776e491..f652016a58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.6.1...v21.7.0) (2025-10-27) + +### Features + +* Add recommended security group rule for port `10251` to match EKS addon for `metrics-server` ([#3562](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3562)) ([de8c550](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/de8c550d5d10017eba4b2c0b492360511fe0c34b)) + ## [21.6.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.6.0...v21.6.1) (2025-10-21) ### Bug Fixes From 0659a8d1cca0d62a7d79d4d4534bf9a2c78eed8a Mon Sep 17 00:00:00 2001 From: Alexis Sellier <3765063+alexissellier@users.noreply.github.com> Date: Mon, 27 Oct 2025 22:49:30 +0100 Subject: [PATCH 938/962] feat: Allow using inline policy for Karpenter controller role to mitigate policy size `LimitExceeded` error (#3563) * feat: Allow using inline policy for controller role (#3512) Signed-off-by: Alexis Sellier <3765063+alexissellier@users.noreply.github.com> * run linter Signed-off-by: Alexis Sellier <3765063+alexissellier@users.noreply.github.com> * Update modules/karpenter/main.tf Co-authored-by: Bryant Biggs * Update modules/karpenter/variables.tf Co-authored-by: Bryant Biggs * Update modules/karpenter/variables.tf Co-authored-by: Bryant Biggs * change variable name Signed-off-by: Alexis Sellier <3765063+alexissellier@users.noreply.github.com> * rename variable Signed-off-by: Alexis Sellier <3765063+alexissellier@users.noreply.github.com> --------- Signed-off-by: Alexis Sellier <3765063+alexissellier@users.noreply.github.com> Co-authored-by: Bryant Biggs --- modules/karpenter/README.md | 2 ++ modules/karpenter/main.tf | 13 +++++++++++-- modules/karpenter/variables.tf | 6 ++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index ad8dfcd979..f283b6e741 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -110,6 +110,7 @@ No modules. | [aws_iam_policy.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy_attachment.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.controller_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | @@ -139,6 +140,7 @@ No modules. | [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `false` | no | | [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_pod\_identity\_association](#input\_create\_pod\_identity\_association) | Determines whether to create pod identity association | `bool` | `true` | no | +| [enable\_inline\_policy](#input\_enable\_inline\_policy) | Determines whether the controller policy is created as a standard IAM policy or inline IAM policy. This can be enabled when the error `LimitExceeded: Cannot exceed quota for PolicySize: 6144` is received since standard IAM policies have a limit of 6,144 characters versus an inline role policy's limit of 10,240 ([Reference](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html)) | `bool` | `false` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | | [iam\_policy\_description](#input\_iam\_policy\_description) | IAM policy description | `string` | `"Karpenter controller IAM policy"` | no | | [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy | `string` | `"KarpenterController"` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 4d3cc08ed1..af0e947e76 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -70,8 +70,17 @@ resource "aws_iam_role" "controller" { tags = merge(var.tags, var.iam_role_tags) } +resource "aws_iam_role_policy" "controller" { + count = local.create_iam_role && var.enable_inline_policy ? 1 : 0 + + name = var.iam_policy_use_name_prefix ? null : var.iam_policy_name + name_prefix = var.iam_policy_use_name_prefix ? "${var.iam_policy_name}-" : null + role = aws_iam_role.controller[0].name + policy = data.aws_iam_policy_document.controller[0].json +} + resource "aws_iam_policy" "controller" { - count = local.create_iam_role ? 1 : 0 + count = local.create_iam_role && !var.enable_inline_policy ? 1 : 0 name = var.iam_policy_use_name_prefix ? null : var.iam_policy_name name_prefix = var.iam_policy_use_name_prefix ? "${var.iam_policy_name}-" : null @@ -83,7 +92,7 @@ resource "aws_iam_policy" "controller" { } resource "aws_iam_role_policy_attachment" "controller" { - count = local.create_iam_role ? 1 : 0 + count = local.create_iam_role && !var.enable_inline_policy ? 1 : 0 role = aws_iam_role.controller[0].name policy_arn = aws_iam_policy.controller[0].arn diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 6d3d042516..b814963485 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -32,6 +32,12 @@ variable "create_iam_role" { default = true } +variable "enable_inline_policy" { + description = "Determines whether the controller policy is created as a standard IAM policy or inline IAM policy. This can be enabled when the error `LimitExceeded: Cannot exceed quota for PolicySize: 6144` is received since standard IAM policies have a limit of 6,144 characters versus an inline role policy's limit of 10,240 ([Reference](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html))" + type = bool + default = false +} + variable "iam_role_name" { description = "Name of the IAM role" type = string From 32599e5dfc369596dfdb28cea120d469c92145c1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 27 Oct 2025 21:50:07 +0000 Subject: [PATCH 939/962] chore(release): version 21.8.0 [skip ci] ## [21.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.7.0...v21.8.0) (2025-10-27) ### Features * Allow using inline policy for Karpenter controller role to mitigate policy size `LimitExceeded` error ([#3563](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3563)) ([0659a8d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0659a8d1cca0d62a7d79d4d4534bf9a2c78eed8a)), closes [#3512](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3512) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f652016a58..00dfa689a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.7.0...v21.8.0) (2025-10-27) + +### Features + +* Allow using inline policy for Karpenter controller role to mitigate policy size `LimitExceeded` error ([#3563](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3563)) ([0659a8d](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/0659a8d1cca0d62a7d79d4d4534bf9a2c78eed8a)), closes [#3512](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3512) + ## [21.7.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.6.1...v21.7.0) (2025-10-27) ### Features From f43d83b7799679398238978b8b07f54a4ec90ceb Mon Sep 17 00:00:00 2001 From: Alex Stan <122723579+andrei-don@users.noreply.github.com> Date: Mon, 3 Nov 2025 23:12:11 +0000 Subject: [PATCH 940/962] docs: Reference removed variable in upgrade docs (#3569) fix: Reference removed variable in upgrade docs --- docs/UPGRADE-21.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/UPGRADE-21.0.md b/docs/UPGRADE-21.0.md index 9d6500b1d4..234d9c11ea 100644 --- a/docs/UPGRADE-21.0.md +++ b/docs/UPGRADE-21.0.md @@ -84,6 +84,7 @@ If you find a bug, please open an issue with supporting configuration to reprodu - `irsa_oidc_provider_arn` - `irsa_namespace_service_accounts` - `irsa_assume_role_condition_test` + - `self-managed-node-group-defaults` 2. Renamed variables: From a19bb8a3f5985a0ffbfc247edc0524463047cac9 Mon Sep 17 00:00:00 2001 From: suauk <40726208+suauk@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:58:11 +0000 Subject: [PATCH 941/962] docs: Fix typo in README.md regarding EKS Auto Mode (#3577) Fix typo in README.md regarding EKS Auto Mode --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 08b05dfff4..15bf384d52 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Please note that we strive to provide a comprehensive suite of documentation for > } >``` > -> If you try to disable by simply removing the `compute_config` block, this will fail to disble EKS Auto Mode. Only after applying with `enabled = false` can you then remove the `compute_config` block from your configurations. +> If you try to disable by simply removing the `compute_config` block, this will fail to disable EKS Auto Mode. Only after applying with `enabled = false` can you then remove the `compute_config` block from your configurations. ```hcl module "eks" { From 2f1454856c8823194ce7cf0cd31f1167675ebab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Joga?= Date: Fri, 14 Nov 2025 18:16:09 +0100 Subject: [PATCH 942/962] docs: Fix broken link to Fargate profile tests (#3581) Fix broken link --- docs/compute_resources.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compute_resources.md b/docs/compute_resources.md index e7dee7660c..1345283808 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -150,4 +150,4 @@ See the [`examples/self-managed-node-group/` example](https://github.com/terrafo ### Fargate Profiles -Fargate profiles are straightforward to use and therefore no further details are provided here. See the [`tests/fargate-profile/` tests](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/tests/fargate-profile) for a working example of various configurations. +Fargate profiles are straightforward to use and therefore no further details are provided here. See the [`tests/eks-fargate-profile/` tests](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/tests/eks-fargate-profile) for a working example of various configurations. From c0ed29b922d22c5950161ddde9c458b1f60708da Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 16 Nov 2025 10:43:33 -0600 Subject: [PATCH 943/962] feat: Add support for node repair configuration arguments (#3585) --- README.md | 6 +++--- examples/eks-auto-mode/README.md | 4 ++-- examples/eks-auto-mode/versions.tf | 2 +- examples/eks-hybrid-nodes/README.md | 6 +++--- examples/eks-hybrid-nodes/versions.tf | 2 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 4 ++-- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- modules/eks-managed-node-group/README.md | 6 +++--- modules/eks-managed-node-group/main.tf | 17 ++++++++++++++++- modules/eks-managed-node-group/variables.tf | 12 +++++++++++- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/hybrid-node-role/README.md | 4 ++-- modules/hybrid-node-role/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-fargate-profile/README.md | 4 ++-- tests/eks-fargate-profile/versions.tf | 2 +- tests/eks-hybrid-nodes/README.md | 2 +- tests/eks-hybrid-nodes/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- variables.tf | 12 +++++++++++- versions.tf | 2 +- 31 files changed, 81 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 15bf384d52..6146850b5b 100644 --- a/README.md +++ b/README.md @@ -361,7 +361,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 4.0 | @@ -369,7 +369,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 4.0 | @@ -451,7 +451,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | | [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed node group(s), self-managed node group(s), Fargate profile(s)) | `string` | `"30s"` | no | | [deletion\_protection](#input\_deletion\_protection) | Whether to enable deletion protection for the cluster. When enabled, the cluster cannot be deleted unless deletion protection is first disabled | `bool` | `null` | no | -| [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# EKS Managed Node Group
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
ami_id = optional(string)
ami_type = optional(string)
ami_release_version = optional(string)
use_latest_ami_release_version = optional(bool)
capacity_type = optional(string)
disk_size = optional(number)
force_update_version = optional(bool)
instance_types = optional(list(string))
labels = optional(map(string))
node_repair_config = optional(object({
enabled = optional(bool)
}))
remote_access = optional(object({
ec2_ssh_key = optional(string)
source_security_group_ids = optional(list(string))
}))
taints = optional(map(object({
key = string
value = optional(string)
effect = string
})))
update_config = optional(object({
max_unavailable = optional(number)
max_unavailable_percentage = optional(number)
}))
timeouts = optional(object({
create = optional(string)
update = optional(string)
delete = optional(string)
}))
# User data
enable_bootstrap_user_data = optional(bool)
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
key_name = optional(string)
disable_api_termination = optional(bool)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
create_placement_group = optional(bool)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
subnet_id = optional(string)
})))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_role = optional(bool)
iam_role_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
cluster_primary_security_group_id = optional(string)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})), {})
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | +| [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# EKS Managed Node Group
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
ami_id = optional(string)
ami_type = optional(string)
ami_release_version = optional(string)
use_latest_ami_release_version = optional(bool)
capacity_type = optional(string)
disk_size = optional(number)
force_update_version = optional(bool)
instance_types = optional(list(string))
labels = optional(map(string))
node_repair_config = optional(object({
enabled = optional(bool)
max_parallel_nodes_repaired_count = optional(number)
max_parallel_nodes_repaired_percentage = optional(number)
max_unhealthy_node_threshold_count = optional(number)
max_unhealthy_node_threshold_percentage = optional(number)
node_repair_config_overrides = optional(list(object({
min_repair_wait_time_mins = number
node_monitoring_condition = string
node_unhealthy_reason = string
repair_action = string
})))
}))
remote_access = optional(object({
ec2_ssh_key = optional(string)
source_security_group_ids = optional(list(string))
}))
taints = optional(map(object({
key = string
value = optional(string)
effect = string
})))
update_config = optional(object({
max_unavailable = optional(number)
max_unavailable_percentage = optional(number)
}))
timeouts = optional(object({
create = optional(string)
update = optional(string)
delete = optional(string)
}))
# User data
enable_bootstrap_user_data = optional(bool)
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
key_name = optional(string)
disable_api_termination = optional(bool)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
create_placement_group = optional(bool)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
subnet_id = optional(string)
})))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_role = optional(bool)
iam_role_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
cluster_primary_security_group_id = optional(string)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})), {})
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | | [enable\_auto\_mode\_custom\_tags](#input\_enable\_auto\_mode\_custom\_tags) | Determines whether to enable permissions for custom tags resources created by EKS Auto Mode | `bool` | `true` | no | | [enable\_cluster\_creator\_admin\_permissions](#input\_enable\_cluster\_creator\_admin\_permissions) | Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index 57cdfa7d4b..aacfd0cba5 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -25,13 +25,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | ## Modules diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index e1ac31cae0..d39a842637 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index e4afac8bec..5b109a4304 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -26,7 +26,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | | [helm](#requirement\_helm) | >= 3.0 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | @@ -36,8 +36,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | -| [aws.remote](#provider\_aws.remote) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | +| [aws.remote](#provider\_aws.remote) | >= 6.20 | | [helm](#provider\_helm) | >= 3.0 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index 74ba3338c5..181b80aa46 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } helm = { source = "hashicorp/helm" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index e1ac31cae0..d39a842637 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 81863c0248..e9783d5e88 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -94,14 +94,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | | [helm](#requirement\_helm) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | | [helm](#provider\_helm) | >= 3.0 | ## Modules diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 0c6561a848..87c70c238b 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index e1ac31cae0..d39a842637 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 1ca9c95095..045223c981 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | ## Modules @@ -170,7 +170,7 @@ module "eks_managed_node_group" { | [min\_size](#input\_min\_size) | Minimum number of instances/nodes | `number` | `1` | no | | [name](#input\_name) | Name of the EKS managed node group | `string` | `""` | no | | [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time |
list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
}))
| `[]` | no | -| [node\_repair\_config](#input\_node\_repair\_config) | The node auto repair configuration for the node group |
object({
enabled = optional(bool, true)
})
| `null` | no | +| [node\_repair\_config](#input\_node\_repair\_config) | The node auto repair configuration for the node group |
object({
enabled = optional(bool, true)
max_parallel_nodes_repaired_count = optional(number)
max_parallel_nodes_repaired_percentage = optional(number)
max_unhealthy_node_threshold_count = optional(number)
max_unhealthy_node_threshold_percentage = optional(number)
node_repair_config_overrides = optional(list(object({
min_repair_wait_time_mins = number
node_monitoring_condition = string
node_unhealthy_reason = string
repair_action = string
})))
})
| `null` | no | | [partition](#input\_partition) | The AWS partition - pass through value to reduce number of GET requests from data sources | `string` | `""` | no | | [placement](#input\_placement) | The placement of the instance |
object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
})
| `null` | no | | [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `null` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index fa940c51a9..aa9bd93d03 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -517,7 +517,22 @@ resource "aws_eks_node_group" "this" { for_each = var.node_repair_config != null ? [var.node_repair_config] : [] content { - enabled = node_repair_config.value.enabled + enabled = node_repair_config.value.enabled + max_parallel_nodes_repaired_count = node_repair_config.value.max_parallel_nodes_repaired_count + max_parallel_nodes_repaired_percentage = node_repair_config.value.max_parallel_nodes_repaired_percentage + max_unhealthy_node_threshold_count = node_repair_config.value.max_unhealthy_node_threshold_count + max_unhealthy_node_threshold_percentage = node_repair_config.value.max_unhealthy_node_threshold_percentage + + dynamic "node_repair_config_overrides" { + for_each = node_repair_config.value.node_repair_config_overrides != null ? node_repair_config.value.node_repair_config_overrides : [] + + content { + min_repair_wait_time_mins = node_repair_config_overrides.value.min_repair_wait_time_mins + node_monitoring_condition = node_repair_config_overrides.value.node_monitoring_condition + node_unhealthy_reason = node_repair_config_overrides.value.node_unhealthy_reason + repair_action = node_repair_config_overrides.value.repair_action + } + } } } diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 84bb8421d2..7f5180fa67 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -575,7 +575,17 @@ variable "update_config" { variable "node_repair_config" { description = "The node auto repair configuration for the node group" type = object({ - enabled = optional(bool, true) + enabled = optional(bool, true) + max_parallel_nodes_repaired_count = optional(number) + max_parallel_nodes_repaired_percentage = optional(number) + max_unhealthy_node_threshold_count = optional(number) + max_unhealthy_node_threshold_percentage = optional(number) + node_repair_config_overrides = optional(list(object({ + min_repair_wait_time_mins = number + node_monitoring_condition = string + node_unhealthy_reason = string + repair_action = string + }))) }) default = null } diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index e1ac31cae0..d39a842637 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 6c260145ff..76ff393797 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index e1ac31cae0..d39a842637 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index 45285c033f..87b6f7cb61 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -75,13 +75,13 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | ## Modules diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index e1ac31cae0..d39a842637 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index f283b6e741..d06cadf56f 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index e1ac31cae0..d39a842637 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index fb04648cea..52cce66c3e 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index e1ac31cae0..d39a842637 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index 600f83d5af..68995f084f 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | ## Modules diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index e1ac31cae0..d39a842637 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index 6c6cd65252..db69a7867a 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -18,7 +18,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index 782544463d..940a0ddbc3 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 9b415f283b..4d148422aa 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index e1ac31cae0..d39a842637 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index df4457cef4..52cf33afc0 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.15 | +| [aws](#requirement\_aws) | >= 6.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.15 | +| [aws](#provider\_aws) | >= 6.20 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index e1ac31cae0..d39a842637 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } } } diff --git a/variables.tf b/variables.tf index 6f3497eacf..ca0b4e812e 100644 --- a/variables.tf +++ b/variables.tf @@ -1244,7 +1244,17 @@ variable "eks_managed_node_groups" { instance_types = optional(list(string)) labels = optional(map(string)) node_repair_config = optional(object({ - enabled = optional(bool) + enabled = optional(bool) + max_parallel_nodes_repaired_count = optional(number) + max_parallel_nodes_repaired_percentage = optional(number) + max_unhealthy_node_threshold_count = optional(number) + max_unhealthy_node_threshold_percentage = optional(number) + node_repair_config_overrides = optional(list(object({ + min_repair_wait_time_mins = number + node_monitoring_condition = string + node_unhealthy_reason = string + repair_action = string + }))) })) remote_access = optional(object({ ec2_ssh_key = optional(string) diff --git a/versions.tf b/versions.tf index 0ebaec8d6b..d5aa3cb1e9 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.15" + version = ">= 6.20" } tls = { source = "hashicorp/tls" From 85a1a1a0eccea95ffa3f7ac1c6047901a1f0a6cb Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 16 Nov 2025 16:44:09 +0000 Subject: [PATCH 944/962] chore(release): version 21.9.0 [skip ci] ## [21.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.8.0...v21.9.0) (2025-11-16) ### Features * Add support for node repair configuration arguments ([#3585](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3585)) ([c0ed29b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c0ed29b922d22c5950161ddde9c458b1f60708da)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00dfa689a9..2c600d50f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.8.0...v21.9.0) (2025-11-16) + +### Features + +* Add support for node repair configuration arguments ([#3585](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3585)) ([c0ed29b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c0ed29b922d22c5950161ddde9c458b1f60708da)) + ## [21.8.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.7.0...v21.8.0) (2025-10-27) ### Features From b8bbba811ae95857aedf4dd8c24f09087e0e00aa Mon Sep 17 00:00:00 2001 From: thrubovc <34124990+thrubovc@users.noreply.github.com> Date: Fri, 21 Nov 2025 15:23:57 +0100 Subject: [PATCH 945/962] docs: Correct typo in `node_iam_role_arn` variable description (#3591) * fix: typo in node_iam_role_arn var description * precommit --- modules/karpenter/README.md | 2 +- modules/karpenter/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index d06cadf56f..a72f55a95b 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -159,7 +159,7 @@ No modules. | [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the name of the IAM role (`iam_role_name`) is used as a prefix | `bool` | `true` | no | | [namespace](#input\_namespace) | Namespace to associate with the Karpenter Pod Identity | `string` | `"kube-system"` | no | | [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | -| [node\_iam\_role\_arn](#input\_node\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [node\_iam\_role\_arn](#input\_node\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_node_iam_role` is set to `false` | `string` | `null` | no | | [node\_iam\_role\_attach\_cni\_policy](#input\_node\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | | [node\_iam\_role\_description](#input\_node\_iam\_role\_description) | Description of the role | `string` | `null` | no | | [node\_iam\_role\_max\_session\_duration](#input\_node\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index b814963485..7e73a3883f 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -253,7 +253,7 @@ variable "cluster_ip_family" { } variable "node_iam_role_arn" { - description = "Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false`" + description = "Existing IAM role ARN for the IAM instance profile. Required if `create_node_iam_role` is set to `false`" type = string default = null } From d3d6697b33dd3ce283626caf4d36fc9ba7a99827 Mon Sep 17 00:00:00 2001 From: Apoorva Kulkarni Date: Wed, 26 Nov 2025 18:45:40 -0800 Subject: [PATCH 946/962] feat: Add support for Provisioned Control Plane (#3597) * feat: Add support for Provisioned Control Plane * update README * update karpenter example --- README.md | 35 +++++++++++++++++++++++++++++++++++ examples/karpenter/main.tf | 5 +++++ main.tf | 8 ++++++++ outputs.tf | 5 +++++ variables.tf | 8 ++++++++ 5 files changed, 61 insertions(+) diff --git a/README.md b/README.md index 6146850b5b..69e5b792b2 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,39 @@ module "eks" { } ``` +### EKS Provisioned Control Plane + +EKS Provisioned Control Plane allows you to provision a control plane with increased capacity for larger workloads. Valid tier values are `standard`, `tier-xl`, `tier-2xl`, and `tier-4xl`. + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" + + name = "my-cluster" + kubernetes_version = "1.33" + + # Optional + endpoint_public_access = true + + # Optional: Adds the current caller identity as an administrator via cluster access entry + enable_cluster_creator_admin_permissions = true + + # EKS Provisioned Control Plane configuration + control_plane_scaling_config = { + tier = "tier-xl" + } + + vpc_id = "vpc-1234556abcdef" + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + ### EKS Managed Node Group ```hcl @@ -437,6 +470,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cloudwatch\_log\_group\_tags](#input\_cloudwatch\_log\_group\_tags) | A map of additional tags to add to the cloudwatch log group created | `map(string)` | `{}` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | | [compute\_config](#input\_compute\_config) | Configuration block for the cluster compute configuration |
object({
enabled = optional(bool, false)
node_pools = optional(list(string))
node_role_arn = optional(string)
})
| `null` | no | +| [control\_plane\_scaling\_config](#input\_control\_plane\_scaling\_config) | Configuration block for the EKS Provisioned Control Plane scaling tier. Valid values for tier are `standard`, `tier-xl`, `tier-2xl`, and `tier-4xl` |
object({
tier = string
})
| `null` | no | | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_auto\_mode\_iam\_resources](#input\_create\_auto\_mode\_iam\_resources) | Determines whether to create/attach IAM resources for EKS Auto Mode. Useful for when using only custom node pools and not built-in EKS Auto Mode node pools | `bool` | `false` | no | @@ -539,6 +573,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled | | [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | | [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster | +| [cluster\_control\_plane\_scaling\_tier](#output\_cluster\_control\_plane\_scaling\_tier) | The EKS Provisioned Control Plane scaling tier for the cluster | | [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server | | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | Cluster IAM role ARN | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index c61b4b9920..62ef9bc205 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -57,6 +57,11 @@ module "eks" { enable_cluster_creator_admin_permissions = true endpoint_public_access = true + # EKS Provisioned Control Plane configuration + control_plane_scaling_config = { + tier = "standard" + } + addons = { coredns = {} eks-pod-identity-agent = { diff --git a/main.tf b/main.tf index 7a6ddbd2e4..a845ce306a 100644 --- a/main.tf +++ b/main.tf @@ -57,6 +57,14 @@ resource "aws_eks_cluster" "this" { bootstrap_cluster_creator_admin_permissions = false } + dynamic "control_plane_scaling_config" { + for_each = var.control_plane_scaling_config != null ? [var.control_plane_scaling_config] : [] + + content { + tier = control_plane_scaling_config.value.tier + } + } + dynamic "compute_config" { for_each = var.compute_config != null ? [var.compute_config] : [] diff --git a/outputs.tf b/outputs.tf index ce5756cef5..027e35d051 100644 --- a/outputs.tf +++ b/outputs.tf @@ -88,6 +88,11 @@ output "cluster_ip_family" { value = try(aws_eks_cluster.this[0].kubernetes_network_config[0].ip_family, null) } +output "cluster_control_plane_scaling_tier" { + description = "The EKS Provisioned Control Plane scaling tier for the cluster" + value = try(aws_eks_cluster.this[0].control_plane_scaling_config[0].tier, null) +} + ################################################################################ # Access Entry ################################################################################ diff --git a/variables.tf b/variables.tf index ca0b4e812e..252f15417b 100644 --- a/variables.tf +++ b/variables.tf @@ -72,6 +72,14 @@ variable "compute_config" { default = null } +variable "control_plane_scaling_config" { + description = "Configuration block for the EKS Provisioned Control Plane scaling tier. Valid values for tier are `standard`, `tier-xl`, `tier-2xl`, and `tier-4xl`" + type = object({ + tier = string + }) + default = null +} + variable "upgrade_policy" { description = "Configuration block for the cluster upgrade policy" type = object({ From 943fd575bcaddaf3b97101ecd7321e79ae67f68b Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 27 Nov 2025 02:46:14 +0000 Subject: [PATCH 947/962] chore(release): version 21.10.0 [skip ci] ## [21.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.9.0...v21.10.0) (2025-11-27) ### Features * Add support for Provisioned Control Plane ([#3597](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3597)) ([d3d6697](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d3d6697b33dd3ce283626caf4d36fc9ba7a99827)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c600d50f5..b62bee320b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.9.0...v21.10.0) (2025-11-27) + +### Features + +* Add support for Provisioned Control Plane ([#3597](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3597)) ([d3d6697](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/d3d6697b33dd3ce283626caf4d36fc9ba7a99827)) + ## [21.9.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.8.0...v21.9.0) (2025-11-16) ### Features From dc4de4fec0d1c50669ac957fbedc26cac0ebe940 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 28 Nov 2025 15:53:04 -0600 Subject: [PATCH 948/962] fix: Update minimum required version of AWS provider for provisioned control plane (#3603) --- .pre-commit-config.yaml | 2 +- README.md | 4 ++-- examples/eks-auto-mode/README.md | 4 ++-- examples/eks-auto-mode/versions.tf | 2 +- examples/eks-hybrid-nodes/README.md | 6 +++--- examples/eks-hybrid-nodes/versions.tf | 2 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 4 ++-- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/versions.tf | 2 +- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 2 +- modules/hybrid-node-role/README.md | 4 ++-- modules/hybrid-node-role/versions.tf | 2 +- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 2 +- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 2 +- tests/eks-fargate-profile/README.md | 4 ++-- tests/eks-fargate-profile/versions.tf | 2 +- tests/eks-hybrid-nodes/README.md | 2 +- tests/eks-hybrid-nodes/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- versions.tf | 2 +- 29 files changed, 42 insertions(+), 42 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 260cc8c757..a223a7440d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.103.0 + rev: v1.104.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index 69e5b792b2..a707dc6775 100644 --- a/README.md +++ b/README.md @@ -394,7 +394,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 4.0 | @@ -402,7 +402,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 4.0 | diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index aacfd0cba5..75030bf842 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -25,13 +25,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | ## Modules diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index d39a842637..7e5b918fd5 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index 5b109a4304..6a6c5563f6 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -26,7 +26,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | | [helm](#requirement\_helm) | >= 3.0 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | @@ -36,8 +36,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | -| [aws.remote](#provider\_aws.remote) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | +| [aws.remote](#provider\_aws.remote) | >= 6.23 | | [helm](#provider\_helm) | >= 3.0 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index 181b80aa46..5950b77279 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } helm = { source = "hashicorp/helm" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index d39a842637..7e5b918fd5 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index e9783d5e88..e623612feb 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -94,14 +94,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | | [helm](#requirement\_helm) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | | [helm](#provider\_helm) | >= 3.0 | ## Modules diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 87c70c238b..941422540a 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index d39a842637..7e5b918fd5 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 045223c981..2af299b803 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index d39a842637..7e5b918fd5 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 76ff393797..8eac03e1dd 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index d39a842637..7e5b918fd5 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index 87b6f7cb61..cce0b4a630 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -75,13 +75,13 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | ## Modules diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index d39a842637..7e5b918fd5 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index a72f55a95b..eb155e9936 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index d39a842637..7e5b918fd5 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 52cce66c3e..ead603dffd 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index d39a842637..7e5b918fd5 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index 68995f084f..15fce74ac4 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | ## Modules diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index d39a842637..7e5b918fd5 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index db69a7867a..f34f5507dd 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -18,7 +18,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index 940a0ddbc3..e99e541fe4 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 4d148422aa..00821b3f66 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index d39a842637..7e5b918fd5 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 52cf33afc0..60525f7a03 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.20 | +| [aws](#requirement\_aws) | >= 6.23 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.20 | +| [aws](#provider\_aws) | >= 6.23 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index d39a842637..7e5b918fd5 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } } } diff --git a/versions.tf b/versions.tf index d5aa3cb1e9..12da3d091e 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.20" + version = ">= 6.23" } tls = { source = "hashicorp/tls" From c41b58277ab3951eca8d11863edf178135ec7654 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 28 Nov 2025 21:53:38 +0000 Subject: [PATCH 949/962] chore(release): version 21.10.1 [skip ci] ## [21.10.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.10.0...v21.10.1) (2025-11-28) ### Bug Fixes * Update minimum required version of AWS provider for provisioned control plane ([#3603](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3603)) ([dc4de4f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/dc4de4fec0d1c50669ac957fbedc26cac0ebe940)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b62bee320b..7e30c1b9ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.10.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.10.0...v21.10.1) (2025-11-28) + +### Bug Fixes + +* Update minimum required version of AWS provider for provisioned control plane ([#3603](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3603)) ([dc4de4f](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/dc4de4fec0d1c50669ac957fbedc26cac0ebe940)) + ## [21.10.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.9.0...v21.10.0) (2025-11-27) ### Features From 60dcc4506399000610e8f2b592f7a68a5131ac33 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 5 Jan 2026 14:20:43 -0600 Subject: [PATCH 950/962] feat: Update Hybrid Node IAM role permissions (#3620) --- .pre-commit-config.yaml | 2 +- modules/hybrid-node-role/main.tf | 41 ++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a223a7440d..e418e76562 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.104.0 + rev: v1.104.1 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/modules/hybrid-node-role/main.tf b/modules/hybrid-node-role/main.tf index 1d14f41b40..71e89aa58d 100644 --- a/modules/hybrid-node-role/main.tf +++ b/modules/hybrid-node-role/main.tf @@ -93,17 +93,44 @@ resource "aws_iam_role" "this" { data "aws_iam_policy_document" "this" { count = var.create ? 1 : 0 - statement { - actions = [ - "ssm:DeregisterManagedInstance", - "ssm:DescribeInstanceInformation", - ] + dynamic "statement" { + for_each = var.enable_ira ? [] : [1] + + content { + sid = "AllowDeregisterOwnInstance" + actions = ["ssm:DeregisterManagedInstance"] + resources = ["arn:${local.partition}:ssm:*:*:managed-instance/*"] + + condition { + test = "ArnLike" + variable = "ssm:SourceInstanceARN" + values = ["arn:${local.partition}:ssm:*:*:managed-instance/*"] + } + } + } - resources = ["*"] + dynamic "statement" { + for_each = var.enable_ira ? [] : [1] + + content { + sid = "AllowDescribeInstances" + actions = ["ssm:DescribeInstanceInformation"] + resources = ["*"] + + condition { + test = "ArnLike" + variable = "ssm:SourceInstanceARN" + values = ["arn:${local.partition}:ssm:*:*:managed-instance/*"] + } + } } statement { - actions = ["eks:DescribeCluster"] + sid = "DescribeEKSCluster" + actions = [ + "eks:DescribeCluster", + "eks:ListAccessEntries", + ] resources = var.cluster_arns } From 8c7acee7484d42a548a2c2438e9de36bdab29454 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 5 Jan 2026 20:21:20 +0000 Subject: [PATCH 951/962] chore(release): version 21.11.0 [skip ci] ## [21.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.10.1...v21.11.0) (2026-01-05) ### Features * Update Hybrid Node IAM role permissions ([#3620](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3620)) ([60dcc45](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/60dcc4506399000610e8f2b592f7a68a5131ac33)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e30c1b9ba..d5d38254e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.10.1...v21.11.0) (2026-01-05) + +### Features + +* Update Hybrid Node IAM role permissions ([#3620](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3620)) ([60dcc45](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/60dcc4506399000610e8f2b592f7a68a5131ac33)) + ## [21.10.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.10.0...v21.10.1) (2025-11-28) ### Bug Fixes From 391b11e0411212c362147e8da37bd2436f250e6d Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 8 Jan 2026 13:24:09 -0600 Subject: [PATCH 952/962] feat: Add provider meta user-agent, replacing static tag (#3614) * feat: Add provider meta user-agent, replacing static tag * fix: Update provider to support longer user-agent tag --- .pre-commit-config.yaml | 2 +- README.md | 4 ++-- examples/eks-auto-mode/README.md | 4 ++-- examples/eks-auto-mode/versions.tf | 2 +- examples/eks-hybrid-nodes/README.md | 6 +++--- examples/eks-hybrid-nodes/versions.tf | 2 +- examples/eks-managed-node-group/versions.tf | 2 +- examples/karpenter/README.md | 4 ++-- examples/karpenter/versions.tf | 2 +- examples/self-managed-node-group/versions.tf | 2 +- main.tf | 2 -- modules/eks-managed-node-group/README.md | 4 ++-- modules/eks-managed-node-group/versions.tf | 8 +++++++- modules/fargate-profile/README.md | 4 ++-- modules/fargate-profile/versions.tf | 8 +++++++- modules/hybrid-node-role/README.md | 4 ++-- modules/hybrid-node-role/versions.tf | 8 +++++++- modules/karpenter/README.md | 4 ++-- modules/karpenter/versions.tf | 8 +++++++- modules/self-managed-node-group/README.md | 4 ++-- modules/self-managed-node-group/versions.tf | 8 +++++++- tests/eks-fargate-profile/README.md | 4 ++-- tests/eks-fargate-profile/versions.tf | 2 +- tests/eks-hybrid-nodes/README.md | 2 +- tests/eks-hybrid-nodes/versions.tf | 2 +- tests/eks-managed-node-group/README.md | 4 ++-- tests/eks-managed-node-group/versions.tf | 2 +- tests/self-managed-node-group/README.md | 4 ++-- tests/self-managed-node-group/versions.tf | 2 +- versions.tf | 8 +++++++- 30 files changed, 78 insertions(+), 44 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e418e76562..fc16b01682 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.104.1 + rev: v1.105.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/README.md b/README.md index a707dc6775..e10940aba9 100644 --- a/README.md +++ b/README.md @@ -394,7 +394,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 4.0 | @@ -402,7 +402,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 4.0 | diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index 75030bf842..0b338d14ec 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -25,13 +25,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | ## Modules diff --git a/examples/eks-auto-mode/versions.tf b/examples/eks-auto-mode/versions.tf index 7e5b918fd5..d2afd5f90c 100644 --- a/examples/eks-auto-mode/versions.tf +++ b/examples/eks-auto-mode/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } } diff --git a/examples/eks-hybrid-nodes/README.md b/examples/eks-hybrid-nodes/README.md index 6a6c5563f6..b8d7f72ee8 100644 --- a/examples/eks-hybrid-nodes/README.md +++ b/examples/eks-hybrid-nodes/README.md @@ -26,7 +26,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | | [helm](#requirement\_helm) | >= 3.0 | | [http](#requirement\_http) | >= 3.4 | | [local](#requirement\_local) | >= 2.5 | @@ -36,8 +36,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | -| [aws.remote](#provider\_aws.remote) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | +| [aws.remote](#provider\_aws.remote) | >= 6.28 | | [helm](#provider\_helm) | >= 3.0 | | [http](#provider\_http) | >= 3.4 | | [local](#provider\_local) | >= 2.5 | diff --git a/examples/eks-hybrid-nodes/versions.tf b/examples/eks-hybrid-nodes/versions.tf index 5950b77279..de04d0313a 100644 --- a/examples/eks-hybrid-nodes/versions.tf +++ b/examples/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } helm = { source = "hashicorp/helm" diff --git a/examples/eks-managed-node-group/versions.tf b/examples/eks-managed-node-group/versions.tf index 7e5b918fd5..d2afd5f90c 100644 --- a/examples/eks-managed-node-group/versions.tf +++ b/examples/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index e623612feb..e530204720 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -94,14 +94,14 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | | [helm](#requirement\_helm) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | | [helm](#provider\_helm) | >= 3.0 | ## Modules diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 941422540a..6d95b316d4 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } helm = { source = "hashicorp/helm" diff --git a/examples/self-managed-node-group/versions.tf b/examples/self-managed-node-group/versions.tf index 7e5b918fd5..d2afd5f90c 100644 --- a/examples/self-managed-node-group/versions.tf +++ b/examples/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } } diff --git a/main.tf b/main.tf index a845ce306a..215f43e334 100644 --- a/main.tf +++ b/main.tf @@ -182,7 +182,6 @@ resource "aws_eks_cluster" "this" { } tags = merge( - { terraform-aws-modules = "eks" }, var.tags, var.cluster_tags, ) @@ -367,7 +366,6 @@ module "kms" { } tags = merge( - { terraform-aws-modules = "eks" }, var.tags, ) } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 2af299b803..ca0e63ee76 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | ## Modules diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 7e5b918fd5..af99edcc27 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,13 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } + + provider_meta "aws" { + user_agent = [ + "github.com/terraform-aws-modules/terraform-aws-eks" + ] + } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 8eac03e1dd..5e0506f4c8 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 7e5b918fd5..af99edcc27 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,13 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } + + provider_meta "aws" { + user_agent = [ + "github.com/terraform-aws-modules/terraform-aws-eks" + ] + } } diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index cce0b4a630..4d85f48d91 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -75,13 +75,13 @@ module "eks_hybrid_node_role" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | ## Modules diff --git a/modules/hybrid-node-role/versions.tf b/modules/hybrid-node-role/versions.tf index 7e5b918fd5..af99edcc27 100644 --- a/modules/hybrid-node-role/versions.tf +++ b/modules/hybrid-node-role/versions.tf @@ -4,7 +4,13 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } + + provider_meta "aws" { + user_agent = [ + "github.com/terraform-aws-modules/terraform-aws-eks" + ] + } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index eb155e9936..5b25eaa7e8 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -86,13 +86,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 7e5b918fd5..af99edcc27 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,13 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } + + provider_meta "aws" { + user_agent = [ + "github.com/terraform-aws-modules/terraform-aws-eks" + ] + } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index ead603dffd..21f55a9557 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | ## Modules diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 7e5b918fd5..af99edcc27 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,13 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } + + provider_meta "aws" { + user_agent = [ + "github.com/terraform-aws-modules/terraform-aws-eks" + ] + } } diff --git a/tests/eks-fargate-profile/README.md b/tests/eks-fargate-profile/README.md index 15fce74ac4..1d34845a34 100644 --- a/tests/eks-fargate-profile/README.md +++ b/tests/eks-fargate-profile/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | ## Modules diff --git a/tests/eks-fargate-profile/versions.tf b/tests/eks-fargate-profile/versions.tf index 7e5b918fd5..d2afd5f90c 100644 --- a/tests/eks-fargate-profile/versions.tf +++ b/tests/eks-fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } } diff --git a/tests/eks-hybrid-nodes/README.md b/tests/eks-hybrid-nodes/README.md index f34f5507dd..e8ce7b8488 100644 --- a/tests/eks-hybrid-nodes/README.md +++ b/tests/eks-hybrid-nodes/README.md @@ -18,7 +18,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | | [tls](#requirement\_tls) | >= 4.0 | ## Providers diff --git a/tests/eks-hybrid-nodes/versions.tf b/tests/eks-hybrid-nodes/versions.tf index e99e541fe4..c5ec0f5c42 100644 --- a/tests/eks-hybrid-nodes/versions.tf +++ b/tests/eks-hybrid-nodes/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } tls = { source = "hashicorp/tls" diff --git a/tests/eks-managed-node-group/README.md b/tests/eks-managed-node-group/README.md index 00821b3f66..296bf54c29 100644 --- a/tests/eks-managed-node-group/README.md +++ b/tests/eks-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | ## Modules diff --git a/tests/eks-managed-node-group/versions.tf b/tests/eks-managed-node-group/versions.tf index 7e5b918fd5..d2afd5f90c 100644 --- a/tests/eks-managed-node-group/versions.tf +++ b/tests/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } } diff --git a/tests/self-managed-node-group/README.md b/tests/self-managed-node-group/README.md index 60525f7a03..7f0b444e37 100644 --- a/tests/self-managed-node-group/README.md +++ b/tests/self-managed-node-group/README.md @@ -18,13 +18,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.23 | +| [aws](#requirement\_aws) | >= 6.28 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.23 | +| [aws](#provider\_aws) | >= 6.28 | ## Modules diff --git a/tests/self-managed-node-group/versions.tf b/tests/self-managed-node-group/versions.tf index 7e5b918fd5..d2afd5f90c 100644 --- a/tests/self-managed-node-group/versions.tf +++ b/tests/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } } } diff --git a/versions.tf b/versions.tf index 12da3d091e..631393d850 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.23" + version = ">= 6.28" } tls = { source = "hashicorp/tls" @@ -15,4 +15,10 @@ terraform { version = ">= 0.9" } } + + provider_meta "aws" { + user_agent = [ + "github.com/terraform-aws-modules/terraform-aws-eks" + ] + } } From d57cdac936efe7ae3b0edbb75340b70c6774d4f3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 8 Jan 2026 19:24:41 +0000 Subject: [PATCH 953/962] chore(release): version 21.12.0 [skip ci] ## [21.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.11.0...v21.12.0) (2026-01-08) ### Features * Add provider meta user-agent, replacing static tag ([#3614](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3614)) ([391b11e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/391b11e0411212c362147e8da37bd2436f250e6d)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5d38254e9..e87ec70537 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.11.0...v21.12.0) (2026-01-08) + +### Features + +* Add provider meta user-agent, replacing static tag ([#3614](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3614)) ([391b11e](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/391b11e0411212c362147e8da37bd2436f250e6d)) + ## [21.11.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.10.1...v21.11.0) (2026-01-05) ### Features From 990050b224f92c1470f1fa2014bf5bb3b6b2f021 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 13 Jan 2026 07:14:00 -0600 Subject: [PATCH 954/962] feat: Add support for EKS Capabilities (#3624) --- README.md | 1 + examples/eks-auto-mode/README.md | 2 +- examples/eks-capabilities/README.md | 74 ++++++++ examples/eks-capabilities/main.tf | 163 ++++++++++++++++ examples/eks-capabilities/outputs.tf | 104 ++++++++++ examples/eks-capabilities/variables.tf | 0 examples/eks-capabilities/versions.tf | 10 + modules/capability/README.md | 174 +++++++++++++++++ modules/capability/main.tf | 222 ++++++++++++++++++++++ modules/capability/outputs.tf | 37 ++++ modules/capability/variables.tf | 215 +++++++++++++++++++++ modules/capability/versions.tf | 20 ++ modules/eks-managed-node-group/README.md | 4 + modules/fargate-profile/README.md | 4 + modules/hybrid-node-role/README.md | 4 + modules/karpenter/README.md | 6 + modules/self-managed-node-group/README.md | 4 + 17 files changed, 1043 insertions(+), 1 deletion(-) create mode 100644 examples/eks-capabilities/README.md create mode 100644 examples/eks-capabilities/main.tf create mode 100644 examples/eks-capabilities/outputs.tf create mode 100644 examples/eks-capabilities/variables.tf create mode 100644 examples/eks-capabilities/versions.tf create mode 100644 modules/capability/README.md create mode 100644 modules/capability/main.tf create mode 100644 modules/capability/outputs.tf create mode 100644 modules/capability/variables.tf create mode 100644 modules/capability/versions.tf diff --git a/README.md b/README.md index e10940aba9..368326f015 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,7 @@ module "eks" { ## Examples - [EKS Auto Mode](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-auto-mode): EKS Cluster with EKS Auto Mode +- [EKS Capabilities](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-capabilities): EKS Cluster with EKS Capabilities - [EKS Hybrid Nodes](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-hybrid-nodes): EKS Cluster with EKS Hybrid nodes - [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks-managed-node-group): EKS Cluster with EKS managed node group(s) - [Karpenter](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/karpenter): EKS Cluster with [Karpenter](https://karpenter.sh/) provisioned for intelligent data plane management diff --git a/examples/eks-auto-mode/README.md b/examples/eks-auto-mode/README.md index 0b338d14ec..d4c54a4445 100644 --- a/examples/eks-auto-mode/README.md +++ b/examples/eks-auto-mode/README.md @@ -1,4 +1,4 @@ -# EKS Auto Mode +# EKS Auto Mode Example ## Usage diff --git a/examples/eks-capabilities/README.md b/examples/eks-capabilities/README.md new file mode 100644 index 0000000000..d15d1c1cbf --- /dev/null +++ b/examples/eks-capabilities/README.md @@ -0,0 +1,74 @@ +# EKS Capabilities Example + +## Usage + +To provision the provided configurations you need to execute: + +```bash +terraform init +terraform plan +terraform apply --auto-approve +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.28 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.28 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [ack\_eks\_capability](#module\_ack\_eks\_capability) | ../../modules/capability | n/a | +| [argocd\_eks\_capability](#module\_argocd\_eks\_capability) | ../../modules/capability | n/a | +| [disabled\_eks\_capability](#module\_disabled\_eks\_capability) | ../../modules/capability | n/a | +| [eks](#module\_eks) | ../.. | n/a | +| [kro\_eks\_capability](#module\_kro\_eks\_capability) | ../../modules/capability | n/a | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_identitystore_group.aws_administrator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/identitystore_group) | data source | +| [aws_ssoadmin_instances.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssoadmin_instances) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [ack\_argocd\_server\_url](#output\_ack\_argocd\_server\_url) | URL of the Argo CD server | +| [ack\_arn](#output\_ack\_arn) | The ARN of the EKS Capability | +| [ack\_iam\_role\_arn](#output\_ack\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [ack\_iam\_role\_name](#output\_ack\_iam\_role\_name) | The name of the IAM role | +| [ack\_iam\_role\_unique\_id](#output\_ack\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [ack\_version](#output\_ack\_version) | The version of the EKS Capability | +| [argocd\_arn](#output\_argocd\_arn) | The ARN of the EKS Capability | +| [argocd\_iam\_role\_arn](#output\_argocd\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [argocd\_iam\_role\_name](#output\_argocd\_iam\_role\_name) | The name of the IAM role | +| [argocd\_iam\_role\_unique\_id](#output\_argocd\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [argocd\_server\_url](#output\_argocd\_server\_url) | URL of the Argo CD server | +| [argocd\_version](#output\_argocd\_version) | The version of the EKS Capability | +| [kro\_argocd\_server\_url](#output\_kro\_argocd\_server\_url) | URL of the Argo CD server | +| [kro\_arn](#output\_kro\_arn) | The ARN of the EKS Capability | +| [kro\_iam\_role\_arn](#output\_kro\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [kro\_iam\_role\_name](#output\_kro\_iam\_role\_name) | The name of the IAM role | +| [kro\_iam\_role\_unique\_id](#output\_kro\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [kro\_version](#output\_kro\_version) | The version of the EKS Capability | + diff --git a/examples/eks-capabilities/main.tf b/examples/eks-capabilities/main.tf new file mode 100644 index 0000000000..a220c7fa1b --- /dev/null +++ b/examples/eks-capabilities/main.tf @@ -0,0 +1,163 @@ +provider "aws" { + region = local.region +} + +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + +data "aws_ssoadmin_instances" "this" {} + +data "aws_identitystore_group" "aws_administrator" { + identity_store_id = one(data.aws_ssoadmin_instances.this.identity_store_ids) + + alternate_identifier { + unique_attribute { + attribute_path = "DisplayName" + attribute_value = "AWSAdministrator" + } + } +} + +locals { + name = "ex-${basename(path.cwd)}" + region = "us-east-1" # will need to match where your AWS Identity Center is configured + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Test = local.name + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } +} + +################################################################################ +# EKS Capability Module +################################################################################ + +module "ack_eks_capability" { + source = "../../modules/capability" + + type = "ACK" + cluster_name = module.eks.cluster_name + + # IAM Role/Policy + iam_role_policies = { + AdministratorAccess = "arn:aws:iam::aws:policy/AdministratorAccess" + } + + tags = local.tags +} + +module "argocd_eks_capability" { + source = "../../modules/capability" + + type = "ARGOCD" + cluster_name = module.eks.cluster_name + + configuration = { + argo_cd = { + aws_idc = { + idc_instance_arn = one(data.aws_ssoadmin_instances.this.arns) + } + namespace = "argocd" + rbac_role_mapping = [{ + role = "ADMIN" + identity = [{ + id = data.aws_identitystore_group.aws_administrator.group_id + type = "SSO_GROUP" + }] + }] + } + } + + # IAM Role/Policy + iam_policy_statements = { + ECRRead = { + actions = [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + ] + resources = ["*"] + } + } + + tags = local.tags +} + +module "kro_eks_capability" { + source = "../../modules/capability" + + type = "KRO" + cluster_name = module.eks.cluster_name + + tags = local.tags +} + +module "disabled_eks_capability" { + source = "../../modules/capability" + + create = false +} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "../.." + + name = local.name + kubernetes_version = "1.34" + endpoint_public_access = true + + enable_cluster_creator_admin_permissions = true + + compute_config = { + enabled = true + node_pools = ["general-purpose"] + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 6.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/examples/eks-capabilities/outputs.tf b/examples/eks-capabilities/outputs.tf new file mode 100644 index 0000000000..3023628376 --- /dev/null +++ b/examples/eks-capabilities/outputs.tf @@ -0,0 +1,104 @@ +################################################################################ +# Capability - ACK +################################################################################ + +output "ack_arn" { + description = "The ARN of the EKS Capability" + value = module.ack_eks_capability.arn +} + +output "ack_version" { + description = "The version of the EKS Capability" + value = module.ack_eks_capability.version +} + +output "ack_argocd_server_url" { + description = "URL of the Argo CD server" + value = module.ack_eks_capability.argocd_server_url +} + +# IAM Role +output "ack_iam_role_name" { + description = "The name of the IAM role" + value = module.ack_eks_capability.iam_role_name +} + +output "ack_iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = module.ack_eks_capability.iam_role_arn +} + +output "ack_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.ack_eks_capability.iam_role_unique_id +} + +################################################################################ +# Capability - ArgoCD +################################################################################ + +output "argocd_arn" { + description = "The ARN of the EKS Capability" + value = module.argocd_eks_capability.arn +} + +output "argocd_version" { + description = "The version of the EKS Capability" + value = module.argocd_eks_capability.version +} + +output "argocd_server_url" { + description = "URL of the Argo CD server" + value = module.argocd_eks_capability.argocd_server_url +} + +# IAM Role +output "argocd_iam_role_name" { + description = "The name of the IAM role" + value = module.argocd_eks_capability.iam_role_name +} + +output "argocd_iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = module.argocd_eks_capability.iam_role_arn +} + +output "argocd_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.argocd_eks_capability.iam_role_unique_id +} + +################################################################################ +# Capability - KRO +################################################################################ + +output "kro_arn" { + description = "The ARN of the EKS Capability" + value = module.kro_eks_capability.arn +} + +output "kro_version" { + description = "The version of the EKS Capability" + value = module.kro_eks_capability.version +} + +output "kro_argocd_server_url" { + description = "URL of the Argo CD server" + value = module.kro_eks_capability.argocd_server_url +} + +# IAM Role +output "kro_iam_role_name" { + description = "The name of the IAM role" + value = module.kro_eks_capability.iam_role_name +} + +output "kro_iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = module.kro_eks_capability.iam_role_arn +} + +output "kro_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.kro_eks_capability.iam_role_unique_id +} diff --git a/examples/eks-capabilities/variables.tf b/examples/eks-capabilities/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/eks-capabilities/versions.tf b/examples/eks-capabilities/versions.tf new file mode 100644 index 0000000000..d2afd5f90c --- /dev/null +++ b/examples/eks-capabilities/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.5.7" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.28" + } + } +} diff --git a/modules/capability/README.md b/modules/capability/README.md new file mode 100644 index 0000000000..e1ec510be4 --- /dev/null +++ b/modules/capability/README.md @@ -0,0 +1,174 @@ +# EKS Capability Module + +Configuration in this directory creates the AWS resources required by EKS capabilities + +## Usage + +### ACK + +```hcl +module "ack_eks_capability" { + source = "terraform-aws-modules/eks/aws//modules/capability" + + name = "example-ack" + cluster_name = "example" + type = "ACK" + + # IAM Role/Policy + iam_role_policies = { + AdministratorAccess = "arn:aws:iam::aws:policy/AdministratorAccess" + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### ArgoCD + +```hcl +module "argocd_eks_capability" { + source = "terraform-aws-modules/eks/aws//modules/capability" + + name = "example-argocd" + cluster_name = "example" + type = "ARGOCD" + + configuration = { + configuration = { + argo_cd = { + aws_idc = { + idc_instance_arn = "arn:aws:sso:::instance/ssoins-1234567890abcdef0" + } + namespace = "argocd" + rbac_role_mapping = [{ + role = "ADMIN" + identity = [{ + id = "686103e0-f051-7068-b225-e6392b959d9e" + type = "SSO_GROUP" + }] + }] + } + } + } + + # IAM Role/Policy + iam_policy_statements = { + ECRRead = { + actions = [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + ] + resources = ["*"] + } + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### KRO + +```hcl +module "kro_eks_capability" { + source = "terraform-aws-modules/eks/aws//modules/capability" + + name = "example-kro" + cluster_name = "example" + type = "KRO" + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [aws](#requirement\_aws) | >= 6.28 | +| [time](#requirement\_time) | >= 0.9 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.28 | +| [time](#provider\_time) | >= 0.9 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_eks_capability.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_capability) | resource | +| [aws_iam_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [time_sleep.this](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_service_principal.capabilities_eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/service_principal) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_name](#input\_cluster\_name) | The name of the EKS cluster | `string` | `""` | no | +| [configuration](#input\_configuration) | Configuration for the capability |
object({
argo_cd = optional(object({
aws_idc = object({
idc_instance_arn = string
idc_region = optional(string)
})
namespace = optional(string)
network_access = optional(object({
vpce_ids = optional(list(string))
}))
rbac_role_mapping = optional(list(object({
identity = list(object({
id = string
type = string
}))
role = string
})))
}))
})
| `null` | no | +| [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created | `bool` | `true` | no | +| [delete\_propagation\_policy](#input\_delete\_propagation\_policy) | The propagation policy to use when deleting the capability. Valid values: `RETAIN` | `string` | `"RETAIN"` | no | +| [iam\_policy\_description](#input\_iam\_policy\_description) | IAM policy description | `string` | `null` | no | +| [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy | `string` | `null` | no | +| [iam\_policy\_path](#input\_iam\_policy\_path) | Path of the IAM policy | `string` | `null` | no | +| [iam\_policy\_statements](#input\_iam\_policy\_statements) | A map of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed |
map(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
}))
| `null` | no | +| [iam\_policy\_use\_name\_prefix](#input\_iam\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`iam_policy_name`) is used as a prefix | `bool` | `true` | no | +| [iam\_role\_arn](#input\_iam\_role\_arn) | The ARN of the IAM role that provides permissions for the capability | `string` | `null` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | IAM role description | `string` | `null` | no | +| [iam\_role\_max\_session\_duration](#input\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name of the IAM role | `string` | `null` | no | +| [iam\_role\_override\_assume\_policy\_documents](#input\_iam\_role\_override\_assume\_policy\_documents) | A list of IAM policy documents to override the default assume role policy document for the Karpenter controller IAM role | `list(string)` | `[]` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | Path of the IAM role | `string` | `null` | no | +| [iam\_role\_permissions\_boundary\_arn](#input\_iam\_role\_permissions\_boundary\_arn) | Permissions boundary ARN to use for the IAM role | `string` | `null` | no | +| [iam\_role\_policies](#input\_iam\_role\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | +| [iam\_role\_source\_assume\_policy\_documents](#input\_iam\_role\_source\_assume\_policy\_documents) | A list of IAM policy documents to use as a source for the assume role policy document for the Karpenter controller IAM role | `list(string)` | `[]` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add the the IAM role | `map(string)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the name of the IAM role (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [name](#input\_name) | The name of the capability to add to the cluster | `string` | `""` | no | +| [region](#input\_region) | Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration | `string` | `null` | no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | +| [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the capability |
object({
create = optional(string)
update = optional(string)
delete = optional(string)
})
| `null` | no | +| [type](#input\_type) | Type of the capability. Valid values: `ACK`, `KRO`, `ARGOCD` | `string` | `""` | no | +| [wait\_duration](#input\_wait\_duration) | Duration to wait between creating the IAM role/policy and creating the capability | `string` | `"20s"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [argocd\_server\_url](#output\_argocd\_server\_url) | URL of the Argo CD server | +| [arn](#output\_arn) | The ARN of the EKS Capability | +| [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | +| [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [version](#output\_version) | The version of the EKS Capability | + + +## License + +Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. diff --git a/modules/capability/main.tf b/modules/capability/main.tf new file mode 100644 index 0000000000..51c7bd6bb4 --- /dev/null +++ b/modules/capability/main.tf @@ -0,0 +1,222 @@ +data "aws_service_principal" "capabilities_eks" { + count = var.create ? 1 : 0 + + service_name = "capabilities.eks" +} + +# It appears that the EKS capability API checks for the IAM role trust policy *VERY* early in the process +# Our standard approach to ordering IAM role/permission dependencies does not work here, so we add an explicit wait +resource "time_sleep" "this" { + count = var.create ? 1 : 0 + + create_duration = var.wait_duration + + triggers = { + iam_role_arn = local.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn + } +} + +################################################################################ +# Capability +################################################################################ + +resource "aws_eks_capability" "this" { + count = var.create ? 1 : 0 + + region = var.region + + capability_name = try(coalesce(var.name, lower(var.type))) + cluster_name = var.cluster_name + + dynamic "configuration" { + for_each = var.configuration != null ? [var.configuration] : [] + + content { + dynamic "argo_cd" { + for_each = configuration.value.argo_cd != null ? [configuration.value.argo_cd] : [] + + content { + dynamic "aws_idc" { + for_each = [argo_cd.value.aws_idc] + + content { + idc_instance_arn = aws_idc.value.idc_instance_arn + idc_region = aws_idc.value.idc_region + } + } + + namespace = argo_cd.value.namespace + + dynamic "network_access" { + for_each = argo_cd.value.network_access != null ? [argo_cd.value.network_access] : [] + + content { + vpce_ids = network_access.value.vpce_ids + } + } + + dynamic "rbac_role_mapping" { + for_each = argo_cd.value.rbac_role_mapping != null ? argo_cd.value.rbac_role_mapping : [] + + content { + dynamic "identity" { + for_each = rbac_role_mapping.value.identity + + content { + id = identity.value.id + type = identity.value.type + } + } + + role = rbac_role_mapping.value.role + } + } + } + } + } + } + + delete_propagation_policy = var.delete_propagation_policy + role_arn = time_sleep.this[0].triggers["iam_role_arn"] + type = var.type + + dynamic "timeouts" { + for_each = var.timeouts != null ? [var.timeouts] : [] + + content { + create = timeouts.value.create + delete = timeouts.value.delete + update = timeouts.value.update + } + } + + tags = var.tags + + depends_on = [ + aws_iam_role_policy_attachment.this, + aws_iam_role_policy_attachment.additional, + ] +} + +################################################################################ +# IAM Role +################################################################################ + +locals { + create_iam_role = var.create && var.create_iam_role + iam_role_name = try(coalesce(var.iam_role_name, var.name), "") +} + +data "aws_iam_policy_document" "assume_role" { + count = local.create_iam_role ? 1 : 0 + + override_policy_documents = var.iam_role_override_assume_policy_documents + source_policy_documents = var.iam_role_source_assume_policy_documents + + statement { + sid = "EKSCapabilitiesAssumeRole" + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "Service" + identifiers = [data.aws_service_principal.capabilities_eks[0].name] + } + } +} + +resource "aws_iam_role" "this" { + count = local.create_iam_role ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = coalesce(var.iam_role_description, "EKS Capability IAM role for ${var.type} capability in cluster ${var.cluster_name}") + + assume_role_policy = data.aws_iam_policy_document.assume_role[0].json + max_session_duration = var.iam_role_max_session_duration + permissions_boundary = var.iam_role_permissions_boundary_arn + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +################################################################################ +# IAM Role Policy +################################################################################ + +locals { + create_iam_role_policy = local.create_iam_role && var.iam_policy_statements != null + iam_policy_name = try(coalesce(var.iam_policy_name, local.iam_role_name), "") +} + +data "aws_iam_policy_document" "this" { + count = local.create_iam_role_policy ? 1 : 0 + + dynamic "statement" { + for_each = var.iam_policy_statements != null ? var.iam_policy_statements : {} + + content { + sid = try(coalesce(statement.value.sid, statement.key)) + actions = statement.value.actions + not_actions = statement.value.not_actions + effect = statement.value.effect + resources = statement.value.resources + not_resources = statement.value.not_resources + + dynamic "principals" { + for_each = statement.value.principals != null ? statement.value.principals : [] + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = statement.value.not_principals != null ? statement.value.not_principals : [] + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = statement.value.condition != null ? statement.value.condition : [] + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +resource "aws_iam_policy" "this" { + count = local.create_iam_role_policy ? 1 : 0 + + name = var.iam_policy_use_name_prefix ? null : local.iam_policy_name + name_prefix = var.iam_policy_use_name_prefix ? "${local.iam_policy_name}-" : null + path = var.iam_policy_path + description = coalesce(var.iam_policy_description, "IAM policy for EKS Capability ${var.type} capability in cluster ${var.cluster_name}") + policy = data.aws_iam_policy_document.this[0].json +} + +resource "aws_iam_role_policy_attachment" "this" { + count = local.create_iam_role_policy ? 1 : 0 + + role = aws_iam_role.this[0].name + policy_arn = aws_iam_policy.this[0].arn +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_policies : k => v if local.create_iam_role } + + role = aws_iam_role.this[0].name + policy_arn = each.value +} diff --git a/modules/capability/outputs.tf b/modules/capability/outputs.tf new file mode 100644 index 0000000000..b812a91c40 --- /dev/null +++ b/modules/capability/outputs.tf @@ -0,0 +1,37 @@ +################################################################################ +# Capability +################################################################################ + +output "arn" { + description = "The ARN of the EKS Capability" + value = try(aws_eks_capability.this[0].arn, null) +} + +output "version" { + description = "The version of the EKS Capability" + value = try(aws_eks_capability.this[0].version, null) +} + +output "argocd_server_url" { + description = "URL of the Argo CD server" + value = try(aws_eks_capability.this[0].configuration[0].argo_cd[0].server_url, null) +} + +################################################################################ +# IAM Role +################################################################################ + +output "iam_role_name" { + description = "The name of the IAM role" + value = try(aws_iam_role.this[0].name, null) +} + +output "iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = try(aws_iam_role.this[0].arn, null) +} + +output "iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, null) +} diff --git a/modules/capability/variables.tf b/modules/capability/variables.tf new file mode 100644 index 0000000000..4458453bfb --- /dev/null +++ b/modules/capability/variables.tf @@ -0,0 +1,215 @@ +variable "create" { + description = "Controls if resources should be created (affects nearly all resources)" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +variable "region" { + description = "Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration" + type = string + default = null +} + +################################################################################ +# Capabilities +################################################################################ + +variable "name" { + description = "The name of the capability to add to the cluster" + type = string + default = "" +} + +variable "cluster_name" { + description = "The name of the EKS cluster" + type = string + default = "" +} + +variable "configuration" { + description = "Configuration for the capability" + type = object({ + argo_cd = optional(object({ + aws_idc = object({ + idc_instance_arn = string + idc_region = optional(string) + }) + namespace = optional(string) + network_access = optional(object({ + vpce_ids = optional(list(string)) + })) + rbac_role_mapping = optional(list(object({ + identity = list(object({ + id = string + type = string + })) + role = string + }))) + })) + }) + default = null +} + +variable "delete_propagation_policy" { + description = "The propagation policy to use when deleting the capability. Valid values: `RETAIN`" + type = string + default = "RETAIN" +} + +variable "type" { + description = "Type of the capability. Valid values: `ACK`, `KRO`, `ARGOCD`" + type = string + default = "" +} + +variable "timeouts" { + description = "Create, update, and delete timeout configurations for the capability" + type = object({ + create = optional(string) + update = optional(string) + delete = optional(string) + }) + default = null +} + +variable "wait_duration" { + description = "Duration to wait between creating the IAM role/policy and creating the capability" + type = string + default = "20s" +} + +################################################################################ +# IAM Role +################################################################################ + +variable "create_iam_role" { + description = "Determines whether an IAM role is created" + type = bool + default = true +} + +variable "iam_role_arn" { + description = "The ARN of the IAM role that provides permissions for the capability" + type = string + default = null +} + +variable "iam_role_name" { + description = "Name of the IAM role" + type = string + default = null +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether the name of the IAM role (`iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "iam_role_path" { + description = "Path of the IAM role" + type = string + default = null +} + +variable "iam_role_description" { + description = "IAM role description" + type = string + default = null +} + +variable "iam_role_max_session_duration" { + description = "Maximum API session duration in seconds between 3600 and 43200" + type = number + default = null +} + +variable "iam_role_permissions_boundary_arn" { + description = "Permissions boundary ARN to use for the IAM role" + type = string + default = null +} + +variable "iam_role_tags" { + description = "A map of additional tags to add the the IAM role" + type = map(string) + default = {} +} + +################################################################################ +# IAM Role Policy +################################################################################ + +variable "iam_policy_name" { + description = "Name of the IAM policy" + type = string + default = null +} + +variable "iam_policy_use_name_prefix" { + description = "Determines whether the name of the IAM policy (`iam_policy_name`) is used as a prefix" + type = bool + default = true +} + +variable "iam_policy_path" { + description = "Path of the IAM policy" + type = string + default = null +} + +variable "iam_policy_description" { + description = "IAM policy description" + type = string + default = null +} + +variable "iam_role_override_assume_policy_documents" { + description = "A list of IAM policy documents to override the default assume role policy document for the Karpenter controller IAM role" + type = list(string) + default = [] +} + +variable "iam_role_source_assume_policy_documents" { + description = "A list of IAM policy documents to use as a source for the assume role policy document for the Karpenter controller IAM role" + type = list(string) + default = [] +} + +variable "iam_policy_statements" { + description = "A map of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = map(object({ + sid = optional(string) + actions = optional(list(string)) + not_actions = optional(list(string)) + effect = optional(string) + resources = optional(list(string)) + not_resources = optional(list(string)) + principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + not_principals = optional(list(object({ + type = string + identifiers = list(string) + }))) + condition = optional(list(object({ + test = string + values = list(string) + variable = string + }))) + })) + default = null +} + +variable "iam_role_policies" { + description = "Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format" + type = map(string) + default = {} +} diff --git a/modules/capability/versions.tf b/modules/capability/versions.tf new file mode 100644 index 0000000000..25ff202c0f --- /dev/null +++ b/modules/capability/versions.tf @@ -0,0 +1,20 @@ +terraform { + required_version = ">= 1.5.7" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.28" + } + time = { + source = "hashicorp/time" + version = ">= 0.9" + } + } + + provider_meta "aws" { + user_agent = [ + "github.com/terraform-aws-modules/terraform-aws-eks" + ] + } +} diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index ca0e63ee76..f3ca2fb192 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -219,3 +219,7 @@ module "eks_managed_node_group" { | [security\_group\_arn](#output\_security\_group\_arn) | Amazon Resource Name (ARN) of the security group | | [security\_group\_id](#output\_security\_group\_id) | ID of the security group | + +## License + +Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 5e0506f4c8..4af5fd327e 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -96,3 +96,7 @@ No modules. | [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | | [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | + +## License + +Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. diff --git a/modules/hybrid-node-role/README.md b/modules/hybrid-node-role/README.md index 4d85f48d91..874021b26f 100644 --- a/modules/hybrid-node-role/README.md +++ b/modules/hybrid-node-role/README.md @@ -157,3 +157,7 @@ No modules. | [name](#output\_name) | The name of the node IAM role | | [unique\_id](#output\_unique\_id) | Stable and unique string identifying the node IAM role | + +## License + +Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 5b25eaa7e8..69c03af907 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -7,6 +7,7 @@ Configuration in this directory creates the AWS resources required by Karpenter ### All Resources (Default) In the following example, the Karpenter module will create: + - An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller - A Pod Identity association to grant Karpenter controller access provided by the IAM Role - A Node IAM role that Karpenter will use to create an Instance Profile for the nodes to receive IAM permissions @@ -40,6 +41,7 @@ module "karpenter" { ### Re-Use Existing Node IAM Role In the following example, the Karpenter module will create: + - An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller - SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. @@ -200,3 +202,7 @@ No modules. | [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue | | [service\_account](#output\_service\_account) | Service Account associated with the Karpenter Pod Identity | + +## License + +Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 21f55a9557..fb267b246a 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -225,3 +225,7 @@ module "self_managed_node_group" { | [security\_group\_id](#output\_security\_group\_id) | ID of the security group | | [user\_data](#output\_user\_data) | Base64 encoded user data | + +## License + +Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. From d416579c20868b075c1799cb1bf6d5b62ee7bb4b Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 13 Jan 2026 13:14:29 +0000 Subject: [PATCH 955/962] chore(release): version 21.13.0 [skip ci] ## [21.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.12.0...v21.13.0) (2026-01-13) ### Features * Add support for EKS Capabilities ([#3624](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3624)) ([990050b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/990050b224f92c1470f1fa2014bf5bb3b6b2f021)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e87ec70537..90ad02a30d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.12.0...v21.13.0) (2026-01-13) + +### Features + +* Add support for EKS Capabilities ([#3624](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3624)) ([990050b](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/990050b224f92c1470f1fa2014bf5bb3b6b2f021)) + ## [21.12.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.11.0...v21.12.0) (2026-01-08) ### Features From 617dba6267162edcd0bf007e1c5fd23e4b43584f Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 13 Jan 2026 09:23:47 -0600 Subject: [PATCH 956/962] feat: Add support for EKS managed node group `update_config.update_strategy` (#3626) --- README.md | 2 +- modules/eks-managed-node-group/README.md | 2 +- modules/eks-managed-node-group/main.tf | 1 + modules/eks-managed-node-group/variables.tf | 1 + variables.tf | 1 + 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 368326f015..681e1a25a7 100644 --- a/README.md +++ b/README.md @@ -486,7 +486,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | | [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed node group(s), self-managed node group(s), Fargate profile(s)) | `string` | `"30s"` | no | | [deletion\_protection](#input\_deletion\_protection) | Whether to enable deletion protection for the cluster. When enabled, the cluster cannot be deleted unless deletion protection is first disabled | `bool` | `null` | no | -| [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# EKS Managed Node Group
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
ami_id = optional(string)
ami_type = optional(string)
ami_release_version = optional(string)
use_latest_ami_release_version = optional(bool)
capacity_type = optional(string)
disk_size = optional(number)
force_update_version = optional(bool)
instance_types = optional(list(string))
labels = optional(map(string))
node_repair_config = optional(object({
enabled = optional(bool)
max_parallel_nodes_repaired_count = optional(number)
max_parallel_nodes_repaired_percentage = optional(number)
max_unhealthy_node_threshold_count = optional(number)
max_unhealthy_node_threshold_percentage = optional(number)
node_repair_config_overrides = optional(list(object({
min_repair_wait_time_mins = number
node_monitoring_condition = string
node_unhealthy_reason = string
repair_action = string
})))
}))
remote_access = optional(object({
ec2_ssh_key = optional(string)
source_security_group_ids = optional(list(string))
}))
taints = optional(map(object({
key = string
value = optional(string)
effect = string
})))
update_config = optional(object({
max_unavailable = optional(number)
max_unavailable_percentage = optional(number)
}))
timeouts = optional(object({
create = optional(string)
update = optional(string)
delete = optional(string)
}))
# User data
enable_bootstrap_user_data = optional(bool)
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
key_name = optional(string)
disable_api_termination = optional(bool)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
create_placement_group = optional(bool)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
subnet_id = optional(string)
})))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_role = optional(bool)
iam_role_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
cluster_primary_security_group_id = optional(string)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})), {})
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | +| [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create |
map(object({
create = optional(bool)
kubernetes_version = optional(string)

# EKS Managed Node Group
name = optional(string) # Will fall back to map key
use_name_prefix = optional(bool)
subnet_ids = optional(list(string))
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
ami_id = optional(string)
ami_type = optional(string)
ami_release_version = optional(string)
use_latest_ami_release_version = optional(bool)
capacity_type = optional(string)
disk_size = optional(number)
force_update_version = optional(bool)
instance_types = optional(list(string))
labels = optional(map(string))
node_repair_config = optional(object({
enabled = optional(bool)
max_parallel_nodes_repaired_count = optional(number)
max_parallel_nodes_repaired_percentage = optional(number)
max_unhealthy_node_threshold_count = optional(number)
max_unhealthy_node_threshold_percentage = optional(number)
node_repair_config_overrides = optional(list(object({
min_repair_wait_time_mins = number
node_monitoring_condition = string
node_unhealthy_reason = string
repair_action = string
})))
}))
remote_access = optional(object({
ec2_ssh_key = optional(string)
source_security_group_ids = optional(list(string))
}))
taints = optional(map(object({
key = string
value = optional(string)
effect = string
})))
update_config = optional(object({
max_unavailable = optional(number)
max_unavailable_percentage = optional(number)
update_strategy = optional(string)
}))
timeouts = optional(object({
create = optional(string)
update = optional(string)
delete = optional(string)
}))
# User data
enable_bootstrap_user_data = optional(bool)
pre_bootstrap_user_data = optional(string)
post_bootstrap_user_data = optional(string)
bootstrap_extra_args = optional(string)
user_data_template_path = optional(string)
cloudinit_pre_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
cloudinit_post_nodeadm = optional(list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
})))
# Launch Template
create_launch_template = optional(bool)
use_custom_launch_template = optional(bool)
launch_template_id = optional(string)
launch_template_name = optional(string) # Will fall back to map key
launch_template_use_name_prefix = optional(bool)
launch_template_version = optional(string)
launch_template_default_version = optional(string)
update_launch_template_default_version = optional(bool)
launch_template_description = optional(string)
launch_template_tags = optional(map(string))
tag_specifications = optional(list(string))
ebs_optimized = optional(bool)
key_name = optional(string)
disable_api_termination = optional(bool)
kernel_id = optional(string)
ram_disk_id = optional(string)
block_device_mappings = optional(map(object({
device_name = optional(string)
ebs = optional(object({
delete_on_termination = optional(bool)
encrypted = optional(bool)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = optional(number)
volume_type = optional(string)
}))
no_device = optional(string)
virtual_name = optional(string)
})))
capacity_reservation_specification = optional(object({
capacity_reservation_preference = optional(string)
capacity_reservation_target = optional(object({
capacity_reservation_id = optional(string)
capacity_reservation_resource_group_arn = optional(string)
}))
}))
cpu_options = optional(object({
amd_sev_snp = optional(string)
core_count = optional(number)
threads_per_core = optional(number)
}))
credit_specification = optional(object({
cpu_credits = optional(string)
}))
enclave_options = optional(object({
enabled = optional(bool)
}))
instance_market_options = optional(object({
market_type = optional(string)
spot_options = optional(object({
block_duration_minutes = optional(number)
instance_interruption_behavior = optional(string)
max_price = optional(string)
spot_instance_type = optional(string)
valid_until = optional(string)
}))
}))
license_specifications = optional(list(object({
license_configuration_arn = string
})))
metadata_options = optional(object({
http_endpoint = optional(string)
http_protocol_ipv6 = optional(string)
http_put_response_hop_limit = optional(number)
http_tokens = optional(string)
instance_metadata_tags = optional(string)
}))
enable_monitoring = optional(bool)
enable_efa_support = optional(bool)
enable_efa_only = optional(bool)
efa_indices = optional(list(string))
create_placement_group = optional(bool)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
partition_number = optional(number)
spread_domain = optional(string)
tenancy = optional(string)
}))
network_interfaces = optional(list(object({
associate_carrier_ip_address = optional(bool)
associate_public_ip_address = optional(bool)
connection_tracking_specification = optional(object({
tcp_established_timeout = optional(number)
udp_stream_timeout = optional(number)
udp_timeout = optional(number)
}))
delete_on_termination = optional(bool)
description = optional(string)
device_index = optional(number)
ena_srd_specification = optional(object({
ena_srd_enabled = optional(bool)
ena_srd_udp_specification = optional(object({
ena_srd_udp_enabled = optional(bool)
}))
}))
interface_type = optional(string)
ipv4_address_count = optional(number)
ipv4_addresses = optional(list(string))
ipv4_prefix_count = optional(number)
ipv4_prefixes = optional(list(string))
ipv6_address_count = optional(number)
ipv6_addresses = optional(list(string))
ipv6_prefix_count = optional(number)
ipv6_prefixes = optional(list(string))
network_card_index = optional(number)
network_interface_id = optional(string)
primary_ipv6 = optional(bool)
private_ip_address = optional(string)
security_groups = optional(list(string), [])
subnet_id = optional(string)
})))
maintenance_options = optional(object({
auto_recovery = optional(string)
}))
private_dns_name_options = optional(object({
enable_resource_name_dns_aaaa_record = optional(bool)
enable_resource_name_dns_a_record = optional(bool)
hostname_type = optional(string)
}))
# IAM role
create_iam_role = optional(bool)
iam_role_arn = optional(string)
iam_role_name = optional(string)
iam_role_use_name_prefix = optional(bool)
iam_role_path = optional(string)
iam_role_description = optional(string)
iam_role_permissions_boundary = optional(string)
iam_role_tags = optional(map(string))
iam_role_attach_cni_policy = optional(bool)
iam_role_additional_policies = optional(map(string))
create_iam_role_policy = optional(bool)
iam_role_policy_statements = optional(list(object({
sid = optional(string)
actions = optional(list(string))
not_actions = optional(list(string))
effect = optional(string)
resources = optional(list(string))
not_resources = optional(list(string))
principals = optional(list(object({
type = string
identifiers = list(string)
})))
not_principals = optional(list(object({
type = string
identifiers = list(string)
})))
condition = optional(list(object({
test = string
values = list(string)
variable = string
})))
})))
# Security group
vpc_security_group_ids = optional(list(string), [])
attach_cluster_primary_security_group = optional(bool, false)
cluster_primary_security_group_id = optional(string)
create_security_group = optional(bool)
security_group_name = optional(string)
security_group_use_name_prefix = optional(bool)
security_group_description = optional(string)
security_group_ingress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})))
security_group_egress_rules = optional(map(object({
name = optional(string)
cidr_ipv4 = optional(string)
cidr_ipv6 = optional(string)
description = optional(string)
from_port = optional(string)
ip_protocol = optional(string)
prefix_list_id = optional(string)
referenced_security_group_id = optional(string)
self = optional(bool)
tags = optional(map(string))
to_port = optional(string)
})), {})
security_group_tags = optional(map(string))

tags = optional(map(string))
}))
| `null` | no | | [enable\_auto\_mode\_custom\_tags](#input\_enable\_auto\_mode\_custom\_tags) | Determines whether to enable permissions for custom tags resources created by EKS Auto Mode | `bool` | `true` | no | | [enable\_cluster\_creator\_admin\_permissions](#input\_enable\_cluster\_creator\_admin\_permissions) | Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index f3ca2fb192..453410cb40 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -190,7 +190,7 @@ module "eks_managed_node_group" { | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [taints](#input\_taints) | The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group |
map(object({
key = string
value = optional(string)
effect = string
}))
| `null` | no | | [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the node group |
object({
create = optional(string)
update = optional(string)
delete = optional(string)
})
| `null` | no | -| [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates |
object({
max_unavailable = optional(number)
max_unavailable_percentage = optional(number)
})
|
{
"max_unavailable_percentage": 33
}
| no | +| [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates |
object({
max_unavailable = optional(number)
max_unavailable_percentage = optional(number)
update_strategy = optional(string)
})
|
{
"max_unavailable_percentage": 33
}
| no | | [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update the launch templates default version on each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | | [use\_custom\_launch\_template](#input\_use\_custom\_launch\_template) | Determines whether to use a custom launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | | [use\_latest\_ami\_release\_version](#input\_use\_latest\_ami\_release\_version) | Determines whether to use the latest AMI release version for the given `ami_type` (except for `CUSTOM`). Note: `ami_type` and `kubernetes_version` must be supplied in order to enable this feature | `bool` | `true` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index aa9bd93d03..e153147953 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -510,6 +510,7 @@ resource "aws_eks_node_group" "this" { content { max_unavailable_percentage = update_config.value.max_unavailable_percentage max_unavailable = update_config.value.max_unavailable + update_strategy = update_config.value.update_strategy } } diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 7f5180fa67..430d1460a7 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -565,6 +565,7 @@ variable "update_config" { type = object({ max_unavailable = optional(number) max_unavailable_percentage = optional(number) + update_strategy = optional(string) }) default = { max_unavailable_percentage = 33 diff --git a/variables.tf b/variables.tf index 252f15417b..d3eab6acef 100644 --- a/variables.tf +++ b/variables.tf @@ -1276,6 +1276,7 @@ variable "eks_managed_node_groups" { update_config = optional(object({ max_unavailable = optional(number) max_unavailable_percentage = optional(number) + update_strategy = optional(string) })) timeouts = optional(object({ create = optional(string) From b704ffc1f7874ddefdcf93d8fc61af82d4cb1b6d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 13 Jan 2026 15:24:21 +0000 Subject: [PATCH 957/962] chore(release): version 21.14.0 [skip ci] ## [21.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.13.0...v21.14.0) (2026-01-13) ### Features * Add support for EKS managed node group `update_config.update_strategy` ([#3626](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3626)) ([617dba6](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/617dba6267162edcd0bf007e1c5fd23e4b43584f)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90ad02a30d..9f316c9de9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.13.0...v21.14.0) (2026-01-13) + +### Features + +* Add support for EKS managed node group `update_config.update_strategy` ([#3626](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3626)) ([617dba6](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/617dba6267162edcd0bf007e1c5fd23e4b43584f)) + ## [21.13.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.12.0...v21.13.0) (2026-01-13) ### Features From 523eb5636d22918cabf211d8bd9f964af9b030df Mon Sep 17 00:00:00 2001 From: Dong Wang Date: Fri, 16 Jan 2026 10:57:09 -0800 Subject: [PATCH 958/962] docs: Correct capabilities static example in `README.md` (#3629) Update README.md Fix the configs in the example. --- modules/capability/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/capability/README.md b/modules/capability/README.md index e1ec510be4..42232ebae5 100644 --- a/modules/capability/README.md +++ b/modules/capability/README.md @@ -37,7 +37,6 @@ module "argocd_eks_capability" { type = "ARGOCD" configuration = { - configuration = { argo_cd = { aws_idc = { idc_instance_arn = "arn:aws:sso:::instance/ssoins-1234567890abcdef0" @@ -51,7 +50,6 @@ module "argocd_eks_capability" { }] }] } - } } # IAM Role/Policy From c2d3b488ce9c63fabeaaded3f369b26a0902126a Mon Sep 17 00:00:00 2001 From: Anton Babenko <393243+antonbabenko@users.noreply.github.com> Date: Tue, 20 Jan 2026 17:07:43 +0100 Subject: [PATCH 959/962] feat: Added link to AWS EKS workshops (#3631) --- README.md | 3 +++ docs/assets/aws-eks-workshop-830.png | Bin 0 -> 72363 bytes 2 files changed, 3 insertions(+) create mode 100644 docs/assets/aws-eks-workshop-830.png diff --git a/README.md b/README.md index 681e1a25a7..74f71cd212 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,9 @@ Please note that we strive to provide a comprehensive suite of documentation for - [AWS EKS Documentation](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Kubernetes Documentation](https://kubernetes.io/docs/home/) + +Register for an AWS hosted workshop: Click to view the full schedule of free workshops, from core concepts to advanced use cases + ## Usage ### EKS Auto Mode diff --git a/docs/assets/aws-eks-workshop-830.png b/docs/assets/aws-eks-workshop-830.png new file mode 100644 index 0000000000000000000000000000000000000000..36284897d8a430782b5349614f7a34da397c8234 GIT binary patch literal 72363 zcmYIuQ($G$w&jV9iYm5kyJFk6ZKo=>ZQHhOI~ChjrO$oc_w~oY*IsK+&oM^0f}A)4 zEH*3v06>tG5K#gE!1#gxBcMTn--p=Hj=&!n2MJAQ005`(g$ z)@8SA7Ov>~>$i$~?8S9eCn=6QkyH>RXduQ7 z@!3a}%}dCbKeIPHvvBO&s`7kKDz3AS-Z=?kM}sk;NuR0^#-HY$aS2I9hYS?ZM9Cu2 z;ONUiAztK4d@^RT*T_oJ;&%d+j=#}FQNqKP=sn?PxO5#qD_=SrkxRByc`-zV*|(-H7qCR|cI zl1iObVOqbMXH+{x#>pfvDXYqwAf&Ye*)pm$VL>RPEI4zwXv^I2$RC${P*LlfU{F#; z@b&^!`tJYr0|JcusMG?HhxRcfIENb|Q2dM(p&YVejB*SBB5SGWimzDiU;yb1x^n4# z_b&c=#k8@6G8m|Zm_%HIaPqxg9pyN*j$+`QB*(w86pz@`r5NbG^wblu6lGHfL#)bV z%ITVySz;t+(J)Kr#2p9b9^Wt*O;^xHmSfRl{YkIzo!Elhz+Cbp5cXA#aO z;Ei=Cmq{x^7|_ZGqIG;Fld)-K0Ie02r01YrOZQYX7gj%2XAL607i3o|zBJ?{EDMK}fP~tZ!jMIeYa9r3##u^kdW4lq?abasK2(oPQNpwM_ zYYMr1Z7HRMgrraXIMS^WNMx&%@kCV5Zjd2TZmA+=E!Fe`(TXxYa?`6-a#A*n#f4Qb zA9BwsD~gF&)n6j86N*;=^gmiF3Eb(NsyMZ1i)16{0c5e1RNCSN$<*oPujGf)U}&(| ziO87-8X0B$*fd}yp5StXG^8si9CUOw^LycS_$i|&CTL~QmsEr@J3<=WZ8})Rb96wz zNlN{@zsG5-hI%qqk%*8r36V+(6UMH24%6hr2>&A=n_Xln5dYMfPbZloOee}LVDiKpXCTN0J1u-gNqhelYIt^E9NrEtmS+3kk6J|{bZgLn)VnRN=C2pr8dUoO` zP1!a*c!Cvj8t3x{PTgQYe3aP%deMJP)N#MLZ>!Z5kM`t8(JHx*56IIXjw=dEap$|J zoH0wKd{h{=mr0U^TZAn$D4KB_mnI7?9Wqr1W7yMpva%XTB(^GUxXvGq%N!t-=#-8T zsUXS8`%AjuX-O)>r9w^6pR^PyXh@{^b4+#^ukTKzACRK0#zP3h;}@{R$~5-dGXm_b zc=XPjJJ{|&9#gvamv7>frt3B*m!7emD3e4|rHq#}E<~1vqgF$EzegFJ3Jx1P4ZlUZ zemP9WC@#;j!cwOy2Hm@lNf;=A1sU^It5sB%WiP@sCtt1>^h&!c3FgTYRzbN06V;WX zNVX+hRH;m%Aa8jP+@~u0mB#I4iDw`rFRT2h2bp2g`oC5wxZmA`C2hBg-EyNHS#GPQ zNF1k5+TuDbWlCcN2F{rR6XX(_R-7XC&hrf&S6_gB|xm}cxtX39Hp9elV4KVsv6 z9~D;EQT#ZDU3e8NBs$?j9lZt+>%v&2qeb9u)BM~R0> zQ#`U*WsbQ2oI@#>UKOaNz}ThqnQ}ub zEN|8{Bx70*<>`YC~Ie}A26Wf(LA{g24x)`Y6T8T#jM*ePb~_5Da}k>u~&4OmetC7 zo=lq$L0b9?FkVef6-h=_@dcqfdc;NE8Z}XH~Rjf5;fu>gHYVGJ6QO?zdGPD(xnASLcEl7s(S(PC4ptGaFlJjP={X1spj4q%~iE8M3zN=XiC1x`#I+NAaZ_5Mi3V(Ma{Sz zTdZoKY`s+R!#WX_E1A7$!maicd4gd|RoFrpE!i8~@ll}QGHvjz6D^peu0#+S+lI0x zX=T6dcndTLj0zbq8p7`(rsNk*5R8Axg%x9ABF4+1zNYGQ|Gmk{ZyuJ$(YvwEGVCa)>4nI`}w^A%Xg;lXZiSJj@bG^})@^ut8zU8`Nu^1ONHJjt- z&b276+<=A^?70w+s1W#2gvwQjfo+p2($a7ymy*{q)(8vymwn(Py_D9*m3E9 zdy>o>!d8mRORSQFS`aCqXbd8`l%KLZg0j45N>3@WAxNz=CU~($4QxjV2?C0KIIa1l=Ht;%#Oks< z&z!{FQs;l*C%yJt@Yx$KAj-z&RNJCu&e^;zHx=<;0zyFWrGn=Et!*yiC@UzErMd^fsu+RBvn!azc0&t99ApR ze9|NFQNAKmXqO>`Xclo#tV_oQdWV`0DlxgyPzu_uPu#Q8onF$Ya+A=ardjD}*hmXP9ky&zq4 zk#T=Gn(K!|qaO3K7)tEM5us9?l&Dl(GC~NonDjyEM(w!p!dy1W5~=-G#45Qkj*?5* zTsW1>|Fe(KKSH~U_8sFoLeHU>8fLOmQV5rIIs4HtSjm#Q2E`YOfH*=)NKhs7mmMA$IGz zq9Ua`wK%Y>M*sk=Gss{75z9asE1Ha2jYwVLKw1LWjue!v(SWY{-krSJYtbuIVW^|k ztZ?y3;#}fX`vrL=-eVId|CY7#tcK{{P12RK!1gJ0-?g(YZXzb1UGxmCo>l&HNf6^n z6pUFxS(T7c^eBQ93tctJpn|as+a^$0h{mY&=ubju-2(KwI5TzfbLD5pr~%BRI}DMw z>?^s9b>Y6e?8ZXPmcFU`8p_BSbiuQ~Z^jab3Tbz-h$D;!0HUaE??R+@3D2xST3K$BY1!EBAA z(6ES-^Xs-EM>?a#8VXe%>|}POa}!E$)0F>N{=Qihqfn^Py!I>b>nlqyi(s*@dA&w!7lsQnxlE#gT?na>+F9&zXXSVQM@i%6qK59)QHArZIw6HQEP}lYy zEjFH!Qp&(_ix9(i1u>Z@{K2Z56o#AJQBsBIj_JSd>y67vA^+UlNz9o$8zzWY zG%?N?4b#uKU6sx(iZ`Jx4q?C^UDz|C%R4*MV8c&;1b~c<@_B0)fIX0NEmPB=tdzylR7)pg6T@epx@yV1bM@duPV1g5Fx~M-x(z!MoIWA+Hx>`oJ zCdmD7Uw1fzsq)kBs~Wu|eU6=Gd^1T-%y0|thGJVUk5Z3?SXzq4MOC-Nouq12AEmY_ z)dWXh*R6D72NzWi2SqgVX*NAw{f&OmP5(P#p-x7Gg%v zBz~dnBwo+#46@x|Bpje+0?i~8E)t|~h}ZJIO|T7@Pm|tKIxH`U01q$uZ{yH7W7>J% z?5C-@<#9?}U-gQh;E@$$AD2(;BPin^_GbN!=266Xh2SwvaCKi|xh`qNZYLswtGW`7 zN0<)b+Q@f(fpH_Vygx3cE}Eox;n@x*HJK->C9$A%>iOCbYOElba9URs_Ur{WZ+v%! zrlf<4D>}qeJx1J+qj*DUx=bjL%$T);K~WB}_UeCcF61%J!octwk&J~1|T!JVgN)RG( zInS&G)a&#j;|}}13H?U!l)-l2>fl5{AW)k;gZLc6XoNwYF-+-n>FG%^{k%v3w82;G z%mF)&;>B=?c8jG$sa6wnFu}ZPy0sO#W7q-lLdep^jLgBtss<~qQaCp^&x?y{PtOwg zp9ImMfC9>#175b~u!2j`;IIO40`Pp74h8el)_g_#*7f^`9}>@4B`zI;)|Iv8>sjfR z)6;ugh>T*a)8fS2s>@!P(~1nWdYTrDbJ!W16v21jzbA%8+p2b2_|4lv@UbwQ5rzi3VS_wLa;p zkE>*?Mi~h&yaN#dIy0PSz0v{PcRs)CDKMXbHS_RS4(Y?vneC#Z$5ZYh? zsTS#bY~^HQQU)@XGUkGcd)TTVZ|qyrQc9R;B4)+nuuF|0?CFwx4bcWt(LQEFh2(KB z)VAc`d0EiK*{HL^;l}lZW%VRwJmixJ?&S;0D*Tro665+}D*DBO^DaXt56w-7QJ9LX zo(Y9*(8%{}92|BV3!bOQ!;E1C0AqHv86GsGaC@j?+-t2HOnq9)?A%XV{jTJ7(asGcrXTd&tETQvXyv0|2m z`7R^y{Mglc9Z@Ho5#itPfIro>r@+_h0S}!cxdp>9uq@PO4ek zAs%zmqs(ho`yNj{v$HM%&%lhv41f!Lh~=5lR2yi-9cJ`lJ-4^C+-h3aRko&~MJ8BA z`vt|*+2&BRpY#@-NP%tBBB<^C5R;8?!2yg;t=iRd+X%zY>S`NG6@GKKy`u1zuCB1% z-3H&$e*0dyrdFPa2oXUvANn-Te6#h4l4@-_E~eZkHEXWD#(O4g1aO--Pt!)Tj!J7Q z_39EAZje_OjL80L=%&^?T=98U99XREnR~9rU)$TN(y1VTpRh;Kc!_K-63eyH+s)0; zipNz`36{z~L=9sOG*cqQEVE;bFe3~L@*SQUOn=0Nny)|h_(qoC?$#Op+RW+ZzBkHt z0G(}Eo=kb5&1}=KA`Z|xi$aA*9G%a_2<5XIF+p)&R~So3)@1heDy#fBofS}K#XGmr-1*0;2>3-T zNtbIX2rM_`86pb`v52s9Nl`og!(1pPH+}DxrO+lnqA217P&uNcB)^*&Rb@@(RVlGq zBNND_uBFVC%)zuos;G~o{}P~ z3+*>H*-)Q9tN#pA7&?J-?WEwp?MvFMbpm(*H~}O95C;9fW^H_~hnw!1a}I>@Z^CG1 z2>4xptLso8#39z5SLPzpfBpcdPh~Sb+&^>x0D@uc&_uzk0nxoR{Evh3?0@hrw2lL6 zi*476e`dL_y2AaduGQ)g);9wQuwX1}T{q+N_2)$PGe3i+$Ca1^jQnePmPFd}oj5vC z^ZIi$%R9ZQU@s*fYkJ?+TV1$Y?kwMRTjhI_^*%qP{7FZbQdJeUYECmIa8%FpT(jwA zu_wQx19tpcw<>Ud4i_rNJ%bPczyy#8R`N7_29Cb?JjSzegf5%`4#H-UFwH(chR4lk zq5vL@n8+prngU(#JGJh-aXY0d-NsZCyBiaj(ayVG#+O;)iU1BsRM3xh3e|0w?H6^u z-6qSz!Xj{I+~YSvZ}nc~wc$;ZuLHffWMn-hDg4gn`mbZQmHxWjCyNZU zSZDuMVgk))G9BN$I4*E9TkHy5-nSG<!TUh#0Js5&0icWkkHRe|?G_IlY!bcQ zI%x*Z@9T@7T_(VCw4iAO8B`FMw_}$3GzGRN_?mr3v#y|Fzx?kn$)U9seLB~{<4=M{ zP@jQ@%d)I&MJBD};rggmiA#_%IwD>tjNMnkfPt_1*tl(R-`axI{W$P&-Se+9HJXiPRRl~ZXON=j`9@NCE>h4yF%a(*J2&SxU!wajMgsIWK5M7-JP2ql5PXMSt;z4WBrglPR`{QXRw{WXFujghBo<@{|#2FzN+SK z%Wb*8`{i~t*G|%2MVbJ^+Ky%2CIEn3KsgjrkM#Eg=h||!b2cdl640baU}4>Qasv8S zH`hnA-A8ul1T4sm_o;}Dp3D;f5LKR6SVBO4f}kR2PYXsK zNPBS&BGw}?i1`E;>)Ls|Z&)~6tl(l?moa52vGw^U+Bk{s4ID}4Rn~CqAjhViqkF&X zBWrrs51oi$L1McB{qzDTU4nv&VF-ag|9({3+po7zW~IloJhz>>?|St25`1CyU%%|Q zrz9IiebLz#+IN{vq_eo!>~>$~d;}v)_ZPwRjF3yGNLL!SI|Lz!pY*)nSUQu}Xty{n zqt1e7WV&EX<~_{jx;_+}``=wdvB*CxuVyf_fCLo8pFXs0ez&}aXv>_Pg$bt}0B=(R zC|&^av0|a!wl_U|0<;7y_FBhzAXgvHpaF$#oc2@DV1hHJicLFx;)P*i>w}?hAc(X& zQ^9}G1{L68JxR6!I-DX*O=OG9s+v#po~s5?U$WDKwQV+A!=-fJtD8_gZ!<1B8}8JO5Jnqqa=BQD0k!~Nv)Qe$9Vi`AVnjUR zjD3xk$v@d{tmhk(F`sRe=SZhhB?{`P-E9_2uIjHKTg};N(c~fgKVVzj(PVH_S*#B8 zkxdW?lKi*Q&WojOgM)+Jdf+d`1 zPMJ>p-Uz_@IAVK!UOVl9&00-?^QgbmEXxg)!0++|=ewp=lqv%<2;gDg*h<1HMu7>?I_?7a+&Ldzwz3XkpQxB2(^uSv=rB>Xik+ zK6<2OW^xN4H6b~I9Ct7$h8wRgm&;LJStDs0M`6pS!$(O#D6&dK5K5H5tB}wGK`&9v z3?t`AHYrkSn5T^km11fe^8yKn856R`R*JdQ~nPG^Q^sdK#H22_pj_tDYxejA;q> zL@DQ1cFcFSJ`*m7W52BDJWt>`AEvTzdS*on4+-%K%8;RSZFNU=-JbJ<1ymVy&76u4 z)~c!4B!~bOyZ(8W7wpPf8-WD9z*coKFuR-Ku}a4_nIhh$H)11(mds}Z4N-TTUiw_E zqoKi0y|ldCWSdDvx1EqYUOrr)h?&u zPSa)nuNVIw!u^MicXiA~89cz*(NW8%kH2*#84HX5mZPnr#ZJ%Qsl(8eVV~nc(>&&# zS7j>wAZF*LwUk_%9l&C<^X9ZdG$zU@!f*-;t4S_YuuH=!RfRq!L!@h`sbK*)n{}Ub zn)alWgsbbvu{enwl*q1h@%E$MX;(MW=&(8&?0Pys`#h#s`Um$GyY7FvAEg|+czVRm z6OGI(l+8*~%tLCt%{4CKw|CD*rK(LUo;uQv7@Y@f4X8fO@C~nS;akSxo4zE^M~< zr0^?h-e7WalsEUV1>|0`JOyt2Q~tNOV6|zW45hAOUgCbQ`-CU(o5jG4tb#V;( zhL^J1KUwIwHlqv%-Y|+0Fht-(bo-JI35_TbUsQq~!YnqmDEbJLFn7KBgKYCuBX2o;U0TX7k zQSV^PqKODs-}jp-B+aNd?b~TN$2H}WwPk4@FLa?C*Ht>7mE*}`)Z$+t-a_7tzfzCqHDh8yamrJp&ZP&f*^YwG2RC_bM_$uWZdy;U0X3 zPNZuo8PeIiE7%;IQYQ@=1B-B4&;~OTkZ$D_FM4cl9i8rth(_L3YvGd(T}s zXeJ{Ml>;W2R5u5n_ib5A4-GwgScWxz_fh-`{V5KrSfu_=iFsZbUNyLZYtF}_$6cA4f0T_#jTLBT1 zm~RO};xv?!5D6DSdqE~r6tc^rBr}5NX-IGhY-QY8}c`D9{U#bq)vetcf3@Y15b+R_-ckQ7EydI zAV$oqY>+Z%dYGp)X2!xpjA8=)%N0|EN{>4NFt%^EGPr!MnPdBH%CCzpsJy+MHVOFV#5~{;P17 zHlMBnu5(t%Rgh($Ppd~rm<%45*WIf+EsycuPV-~S zZujnQ&-E-WWqF=_m-~tATwmR{hkf8mj1@VYkyzbvn|Q+6y}VCJ{|wq@Z52JU79lY8 zP{L}SS6%J;5`VepTgA10^#Mc<3l%1;f-3Y=t33J~{u#T+wjDfk#n$OJ0^?)`} z{GeZfiP|x6SOW=w6L&F7+_8FET76MSII!rh-%s+i3y1-O>OzQqx(fVPn%i-7q#mp| zFH#8Dc)6<8?|hGV(Zlyx!IIYZID8eyA1gBKN1aWXPRpk#jUxgk(vAO!<=a_E?vCcd zj;7|@WG%(NuZe+iRCoVZNt=2HokY0MKiS(4yMEQg0uPB$IW@XHTg$z<4wh>QV;Py& zjf+qmH!$wr@G%-fQx8B$Lsu0Y;DGk?Vv0OR!`#YYjicxGJ}z>Ge=jgImiE2a>#J7l zporNpICf$8o4xHwXT*e=JA0bP4gdAr`mQjD=@~4lSBL${XyP)Ud| zIhWj%T||1Cm%RF${@FH~J(7?t`!^9|v%Il*LWZpbi~~zRfq+Q0i&{`qVHp(p zp*w-$HmVj(^$bJPgi;SKozhX`5+phxOA1-q5uURCAY;nb6Uk!a3N4;%xPC>9NXq&U zt;0k}ync88U;JRUR%^;33gw_TAp9TN%n+@J$X}(?=TwX+reVgYd0dFU#D)=u9hDXG z9?#>Jt2%R7pfSVC(y4UqS8|5i#DH5x3@~`^8$=C(e2$@Q&Uj0bz0FMbVui*s|*6zdL{0E=!tkJoe-ot%!0+8NH;>lJONT;B;Q z4!tGU15qA7t{44t& zmYW?MIC2@z*Xh=n1fvBGYrE%_)g23ux8V3yJ+)rHhf^H-?PkxHyv&P$~Rh_n-s ze81A{eTcQMJ#j$Y$Dde=udj&+W%-?#w^ZKIZ~?#LHb^scTbAiFW0N%R6SR; zpD>kD5Ij-+0f2&jLQh2i0EkH&aY_kY~+vpdre zg1r`_2ZUGqPqL*J34M~>4ohylzDY(bqPU_0hvRti*5DPojjBt#>FvibqfldYM{LY~ z<&$x2FSI;F?5XSVer0hb&ZIZ`t1nk{-dL}%QC?PR&b0vLt{KVC_nhMuVLNz}YD96D1$i7INfDU}=ACWs%*sZ~RvM52Nxb<3 zZikEn!Z3KMa2M?Nx0}^mH}}Qyt1N*p9bLldGz7XMopU|@n15x$EtGc~UN2!se z@&wLT=*n_g7A*oV+oI|~#-&YEv_KSXIz(8pR~?DaPT~j4tYeZ~Y3pVIZCySx*H{>c z41MbCZrkBZXsyRd@Bu-W^)Ohhv(GHOGIxpNA~_1Lr`!+sz zu~5wL7#>WqY%Mo;zXj*9WFq&u|J2=VaBdw>uDMqf9t$!U+=ROqN8*22|6sz|em=LZ z`G<}+J*1I%AVaUdcA)t(gzC-zUgoe+w5P7aZL;;Z328^eVIw$a1N**_L__9$p2>OB zo-Q+}V=T0%Dvw;gTre#JLbFJG_%|e z@@aJ5?m*xXfc)$+m7bn33k`%F*oTa*)~o#37gL-l-mZDT!*>N{v{{u}HecaOa^bK%x@yRthodl-ok zLqmtO*li^Mv)gL&JY)l#JehJ!IlA=wrtz1qEG%j}iJVWI0|c(5r10_l%GM1~SUadU zF}q(#CYIV8FifQBwwN#x=sUfj&u|zN1t1Uqg`iAh+*z)zt!)<-PQj#gd9R7($MO27 zS@@nx`|o;nA~$ud0d_1Cfe7vL65C^D9RqcajO&EfBNJoaUy+eX!R7O?q%E zBhs}9`$rNnZm7g-OKwUD?yqGbQah0-%9NA=EQpCwl0lZ2v{W$Yc}6_YG!a|t%5 z%B&P&{9LQVIg60uIG-dtA{=ZcALOTCHO4RmXVzlqll&X3mQ-mXh3?o_)!f&zVl+5%+RShWI#VvWDJu~!Qno6`Ch$7pLQ~y%t#Ty9Ly{~*Wbs*mY&aw+G5c( zY6@6eM#2oeueHS^=TiBe8?~X+Aybe$NJVJExs_vEtFU)a5c`j^${GUxhqm*<<|mw% zw1ox1HR9gx?cJ4A;00m#Ltg>AZg%FQO8w-XOyyAIQV482y{ zjiVWvKkvML>t_FA=w^j8Sgh^!y1)Up4V2sAK1_ICpnopU8n_n8w_n^v&I$tYA&Ffv zB1>y(yv`bW9?VSzluwZQW_KUkD-*pkwF0YOp7!#|DCKzVbPtASpZl@;Xaa{?>(V+lfn_ZT-Z* z=D8Wtz47l09{dk9ZkhRS$rwxw@$>Royw&H~tD|v9SvyoSh5eL{iz}?a1w(puuP|BQ zoA+ufF;vIlU#oSJiuIgH$z5%5K52l;P5(H> zJFe<70Qb5yLHnSDC?L{bvIhwWTb+~^kisOamiJuLp7nJ(4^I8Moo}}ncJ77>s6zDj^oGjiO=1?^Pjsm+da_;;JC%6DK5Tk+DHLGWN-k?2V4jRcI%Sv zG6m>3Msj6$Q`M9F!6-=-K4I4Av_t(1$9@2saQ!AWYB}_LQyZV^(usQt#*4s1(=+tt z>nu-pXl$C!S=tGo?vto8d5+6;@bF@(o#z`{;UJNZsUiRxdG@a5^E2gYQ(fSGN-}oq z2S^`o_)iVN!GZBa*)yyX8TEffkx&}{ZWsHX*~KN9jy?RPFZvp1C2lrfPG@_A zxVb&2kb3^!#V?NHu6gD;ULX$FsmN5VuP67?>%NS;JI@LnW!d*?ybZDJbaXWF10}fa zr$LeOJW8uASJUp%mU2!$e?S329x;taC#^QS^~dw(sQeUUAR>xBtXi#RS9L6AT=#ev zskM)-O=&F58%Vg1t!}CC^mbKt_Uk>9r!M4ipkM`5`C{pg@EU|#%d~saDh_JMmt! zYDKrxDy6#a_$ZB;8MN9qtbh{j;~mz`Ar z+KTI{H?~r%DX{SBTQ`^KxogM>vC}tROnazd;>=DvvFzsGe`2@aB^q+aN_1S5wtwca zoZ0HESzNdI=+)?h{PT7|A?N!yCcBY5PRliWnr7QDW zg{8*6B&QhsH6|$5oJm00dx11&byHV$$LF8IgdZA&EiO$4t}9Q854F9AiRuim`_nHc z`YOFnrdqrFfi2I!B~hhrvVi3|u#lV}p?M ztqNsvy^bKl28sms>~dJH@Hp{Ay*42a~EBp3agHmG-lo+`$VTHpJ! zoCx3MavQC`Wcb#eavk}fuX!piXRI zs7#@CL_;^sf$B4C9aYrw*>C>oQQJ)jp9jSVrn2Jt>2%>+wQKHjSRs~4nLT6BSkw05 zhrfu*kfwgO%%ca?6Mz`Z=A4VJZvC~kaX`a>oy74(e00oZie(yk9{2=qo7#T+S@oj` zOl#3;@*B~ZIG4}Iwwqu17BPyeJ@y1YV@{o&fD`eysyNh9>_)w}xw>gGoq#%R|AU6P z87&}5ZdUqh6xU293O!)u!c#8Z)T9a!MHL^+Bo-iZ1}QabT!rrd>&=S=28r1FxP<{9 zo!MfBj({<@o3v-O-QIeyuv|w)(khdo(&&_YORfnQFjF*&JL{Y^eiUKr_%qrF|H2lM z1t83Ntig_p=W<-04v6&EilXGMw&yZ{hjzs)=lwh=xH|}e=YB}4D5fb0bl7WCGy@p7 zJXFQ%Eh-bX*@0NtT)!IZv{#K5wyx8zYvO&p`1*|mQ}o>3UtTa|`FsT{0&oPO4feG4 zt9%a~3l}AsWPB!BCz47cy`s~gqUv|-keZrny$;hqg9(Zh1uxRqY)iV-0GMFa;^YDA zvAa}$NeciqpL&PA=2q_e22UW6zzqkmT{6V)q^3sn|9UD_*Piu%IFs)Ez6B;yt$uR? zLdc`r$Km8`hR>U(5CFChat=R`z`m`}ch=5Y$j@m@en#&;{Cjl7R^m$JPse^UnIYSH zeHKarVnPMPo}iUrArM;g4i38_d8qwI`f1r3e_L72!>TKateY5pO1 zkrOT+ij{C#Zq^x)l#?)wFhJBSzz&!wmwr@Op@Z>J4BBrZQNkV#BV-5dO$f-&BL6{I z4p+pk`pcSGN*Q53{t;tUER7PqR#*??FT(?(a?S}@V4eVS*v)&mEP3IyQNCAe!gnmP zp`|t@?z5LK?kF4;d*!eNtJ3&O;dga`s3MxI_@|%0v-5Erk*D$K+QWlK4A7d*z>Tu! z8z>%Ah6m#Xl4n?8P+HCnQ#~q#Fpmu$Vi})?2^-^&G_3-t9BHSX37GI z2t<7DXhSg}#S8zftN;~IgL<{yb%gcLaR-ben!g}VqA(3-rT_skze151K8fab$fpbz zt+0m~`)6kAHg$nR`)?RjS}v|1wWY2nH6)S##{-7memxo~2*EmE?_3d)*c^%ct((B3 z$)j$#eTu&NK(tDwnTvv$jtl33+P0~5EMTq#b#!7CGM&Gasy|NSvX*&nwp>?W`*tlF zv8kyPJyF?jbWHCqOkkqY(5)#@R(DM;_v%kRXYY)NIkD^Q8(nR#>J~fIYqi*p@~YXz z^@j`|M@4-ejJT<*w_fd#^faLnp*?#oemqZOF;!V!0H19YQ39N9X3FBX#c6nAapR{T zhSDO%SfCJC%}rSC*0G;Heh71sh^Oh)0Bt;oA6qsRGT@b2Y%mZ6MCu+=s&$fNX&7W_ zXby;fIF(>muZHsFq-1w8juHWBYh>Y^4zE9hiu~qkD}Vc=)(@?1s4_YXG+N~eYFveExH(E1PY}DwIDg^}MKmL>p62T9-7CYWk%ED)-=8{KuLUAy?afyfrn02SQ&ZD& z^3HX%Pyp`in$FE_omC+3k2{(sD>@v^Dtfn&K9454yvhOD(IHg&$6_fZJ|{JIvYz%$ zS5grgtfIO+(&MXdF5sD@$iX zc{wIAJL7WaQ|nfC6HVGm0&C=1y`y zU&cf?os~`{H}`{IAtwYf#}w!)G+7G$A;B}3{2F?F<3ciu)kCS+ZoI4l#!=7niyxxv zu&~Ggqcot3xwd>fA<@{WeTXe{U~`4<=?QIc?fxM+)Z#zl==C#7Y9kr3oFhPhQJSE5 zXm9cGrIThuqh|^_eNk^a*ZzYdJQyZefyJMlQx|42FsvhC*5P2{e=TbCXprD(qhSTW z3pbAyYW+NblZTFu!ElKA>i zBk|%KI6I`|GT0Q5W8Sf`iGqP>>*1fpHZc@&K{UQrz8&ySrPVxVux_3GNQXt$1;FDPAduc zLLqquiLCx_-Bp6@mRcn7Q6mKhrK{9Dbw)A#R+4BagMbS8Cku+JVH}tp$wF7zjJ!am z$SpihN*GKxQ}K`;GcyH@)4$rK4b_1hGzs|!GGr>4g$@`vh3oG3Q8?9``!F`{jmV7QQ5<-L+ zYqc2@-lVA@a5^$(Q;z`hbn6B zgQAyaH0J-ULy%;rM)@FpNv2h${SDPTq4>)*{}cnsby6Yvx9$bN@wQZ^qOMLL(@0@| zV=7zJvGa+MPl66Ea;p%vkZpROCh;d3p;%`#ZNcaKe#K)y z-9job!c~9A@;a6=rxfa=+iq*&-Hs5Z1haU5;A54`4^kVkp2rbVFPC#-X#e#6tNS-q zFkV}$z#l#rxFb7}zYDqSDuB;KC{wal+_uFH$4U_lf@38*WRtw!a+qoxE z9UZFRhzRU=f?O&ohCm$6V>`ZD;+a&u@5!thR zbK7XPbX=XQAd%8yqSw|#lhi3{etK_*ws>MM6$-VgxI<>d2xo>#s#jV1E z4y|HyyB~jy{WWU5MOF;+J&s*dF^)FB3a(8Mj(=>`UnfCv`(u20t;-@~{ z@Q*^SWv79i{Uh^6)r8QLc@{&N4~efxl>3FFQHV^*vsI$o*omPk1Kq|Zwf*!Yj19$u|YUl(+k`^h?RI`9y} z6eLY#95+9yQY~&D@5e~?1-WndJcTT1R;HY`-&L*ExE}2|iq0=4^hK+)jn?(1Q4Qia zpVW{-n{lAvDPLZ1nb9yw!P`3$Na5cM#o$rVRcLk^E#$lsKn%P#wKVj7s?ydMXRl_M zM%_;Z4sIW9)#ctJuSkQQ;_%r-hnUmTzZOjWb@s8&wg;9byeGf7hsk;;%$b}ypt;5; zeo#5bQQTUtHQaFM5uawlB4oi2kf*NIVfee?L6t_LB3q9jak;({O<&HWPKj!25;q-M zN+6vAmbFPPqr6nu6I#;i{00kpl*5i!Q(&Oi+7Z!Kt{}ov_-*?~goxyuLMs_oLZe)L zIHsK{3o1+)dV8IWJe23Q)d?)wwXUDUmc6AHCEYvEq=*XUkP{=L9q6K3W8=;5a{83- z-p$nwIs{D;=Nc^niGcHA6xD!mvbtJ%r*LUzD2}kvcYob~sM&4%%jdOFR6HN# zQ^39IzwH?`%s6E!GHZPYnXhs zV$I4Y-Qgtr7Dg%A9~ahy>ZD7afq9NmxP=N8ScearKpm_B56xUht1i?1^UX$7MNQxS zE{Un*?VnIiwScx=jTRf5-l3IZ4{lGGZ)UT;+iNy{iXS;Ir`JRL_wsq%%9QUmTx;&$ zV``!FzJ~8zh}evdyns`vd&592p3B29q+;-MAFP>$InYQ7ra&OQwKH5*>PWVo-pz}Q z+svLkCfctQwdZ%w?^F7!3+@kSf|qyr{dKFr@n`Bfk<@8pZ*{&te@lgKA?b~rw1p@SfpTGLLRR>vy6eZu@F8jBLSwgB9wJzAc&VR#(iDmL z-<3F_gv|^LTzfGBBo*%B&vtktMbp(vSui>4i|_d0eZe4*gx zJIi-MTop&`CrSBmj?PttIiOY(*U?f}n#u4+(4AMOaI*7!Cd!Z@omCbFEkhpJYOhIm z5f-6e;_ThA`J)ix9|<7WAbq9=;SL-LXtb?y=AVM_k{em zjVrE51vAuc^Wk+22e&(4?E!06aa}#@2;B%9n=Zl7tA-aX4EVo&LXg6Hp}^gZhwhCn z7R&=Ir{DPBl&E@PmK&{TNWK%-bbM8?Pa+_%#k5ktB8VX*`4haLX`e)+C~rybY8syR zL)Z#on1G}Wc_3Tu2}x5z%UGTD7o&>$9>peXVKs zFV7+CY@cO|qT3OmU4M{_vFD{(?4KliRW}^6?O8u;*{Chf$3Li&5+hmA>xPdcCRH%i zbca$a?WPV}uz-v7beGb!TfPu=xE+UYRi~wu0^=`D!fmW+o4+zOa-^~0A>^ijhR2@Q zyN^l|teL`*CcixxjmF4e(D?P>HglY+d1z*8in3Vf@?Un|4B=|m%Jwa(@D8edBC9Id z-**EjOp~!XhG&D`o8*^`pX*_(m|il5RF4bNPK%Fr1|ntaR_W7*R0=8IXrp#`1yUpu zL*+m;Jk1eGs#g&xB);)O=qdys8p356h`^k3k<=*udGzQE@Uf&>@-4wzI~29FRx0@j zqUm|KR?rD`Dy&?}(MM>*GKqU=i-qeN{V%@p#3Xg^BGxG`d}lbKmt9!c!hcC%rE?tv z+i23wq#HhihP9-Y$Q*rz2?}Y3@dCviWpzJ^5SfTa{0*V>p{z;|fd`$p>f5HCV05mIS)>X!MF1t)B1fo7JAlS!+ws%uW*0O5=Q?>xQ(4n+^7u_>W$iVhm$_kO+s?`o zHnd5T0cTs71b)iWUL|17JVH{HKI$D3bpc&bRYiJo58WrsZmr^2} zwPC`ZiBuxp?$rs5ka`=FqBBT?IHWXmWA$m{45QX8DHZhbN58H#Lqxrexj(W>tIa6I z$1BGu4zY7JyEnDFdJ&E~eW|a_BY`8iljN>NuKgC7Aikyff?7^)*7?P*XA7G37G9bFg#`1<&NSxS(r#5*rH!ri zqvx<})SP_Amp9Xv>87*QnaRbh+*IiQU>Yg(v+v`nN~dCKQZ{Xs?IQRfykzqN7!#Dz z5}`nSVJLMxCk&%*uI+KT%$#g}l%;lSwHEt1uT?kfk@?Y*eKrnYmowb!0Ra!VyYuJy z?%b0THztF|vwg~P_390ue}70Li=z+_Z+JuHo`?e9Q}W|4^!sv*h^khhR#PJ<65n5+kp;#F1)9(op0~#YezW6Wla;-wX3c}3K&B% z=&(aND|}6-i`TdJQSEK3R7oRz$#2A;gX1ClapDEN>%9DNmFo2(Or8sL5~L}+9fCrG znjU7j5-ObKyJ>!Rz-7RH4Em+qLAGwpi5~;|i)^!fVE?+=-OceAGdu=9Gi*kaW~E}- zxFs_MDoi~?6pP?}MG|1j{Sc5hXS@VH_}hl86C)XvrTCTxQJ;>_L^v$nA*$DE4Vkca z-?&PY`^wLTi7;pNeUT9dG8uV zp2|c{Zm4Z;1&TDI7y}~PxOIhV(u*u%7_)%?cSJE0Jc5bj?^(fhz-hOrqcLaNl%WG%brYqK&cKpKjV|`I&5^F{8gzTu-`(Hv$^BK?YMeBn2&EY?_QNWmtXGX;_v4>w z#2%Vx7zzOqr_V(>L`V=Ipv%m1eICBfZK+2U)s?x8KaKBau^33>@>wrpBX$8jaeFBC zL`R>D;*9SH4oRKKLKjA{{6~KgXu1(uM4My0)*l}IH6r?ivX7>V5@+u}U7I&Ix_4bpNSPH4fK$A|Rlq4h3XPT&la%Bg za)c+%+4mI`8g<-Jin2^*%Zzr*ZvMZ@|ONp~(E zGc$Ei2`iUtZ5*6Q)V_TSZwR|0PoN=bc|Kpbtv($*?Q;=giAvSnh6#(prE zVI>#1F;pX)tWIqTzDDC-?<;*yEGxsCuUP+rTyKuZXr1&oANixPF@ZB*XMrSY$NsO; zfhQvH!Ks}N3Z{+%t12BufV8r*iv%kcoEFd{e567s9Hf4gnw2>d8H7b@!@>dQL z#2DQ&yUV@*{(BRZb#J!V2p|VlWqaQt(yB*!8g@r!x8J4PHeH%v)U+6n!|L$|_FlJC zI4pS+`{-+Iyo@-<-5u*tl`k7k=CXS~$<&vCyAPs)`Mk3BHwPDA(rXYLoXs&l6aRk0 zy}ox-`KZD?rB7h~l%A`uk3KQFql}+Q0QH72pQBd>0qDY+f*DM6X4z*EGFD z7co-kUv~`(Y|@V_i?dtzkN){Ik#v7YIdE>k^QATLvO`ucPqq?%W^i~8rb1LKM1hlL#tYLI)KrxRY0Gx?*a$>Hh1rete{>F1M(o^;%x4?86#iN zb?efcQgG+LIRXow=%|0wTL8Llj^|ziJrmEuVGaFj#qUb+tq^hN^%VA(dKskr(T}A4 zPPQ~zX+J;5NP3QqF8%{|;KPf;la~E%o!h@wWtUJ<^V4sdG7JY3_+dB8#Hv*AE}bjI z?njh(pkm$3idMdu{Nt>!f|23ae!|I55RYOWOiPUxq`KAk6!i(EfQDivD0`oUM$9Jh zTfV|4&~S1Q^5>tsX+))>X&R?Nd?yS|C~k-?k| zgoL|T%`9{;>>Sn7I5g4uTy6ubtyto_Ke>N=>FqOoh!4o^y0%~X<43+->Ui;Ech8Ow zK;B9wsXsy82dRsTVt(kB{{;j+9@3LgUM_lX zZN-x52he7EN%Cq|MGrLHSKj*@aQE98qq`6SycQ!>j)%xIF*T+{NCY)8=pm>_nxgEs zMhz==dwH49;eF*h^?1hm=_SzWhI0IVmlZ26mY;)}80}6=Ah}|&K|HZ5?eFmr;sXPf zvwJw4S7Psh_qsqdYbz_E>)BQlDP(BSWi}=7x7T~^xxwV0JHIvwuh?=DR#w?r*+3>n z&_dTE5w2kGj@y0r^*1^1-~xg6nzK^H#PAp6l&i6!>*AcNa{e0#UW!aH%x?AIv#Ep> zT{%E;=$WXhhaRUo2%Q+HRP9=@&92WGyAcUN^-CsJLN-cvm zfawy@y6z{W5WjXaH~qX+g>*0P-eK+#_Lb`usHj(4OmOe^1IY0cEmP;E9|_Kj>Ljws zm?Rkh{Yf#ZTX)x+Hy#&_n^!8nL=EWbrrp1ruOgd7^Vpw_{jR%-*p`gL<*{8Wu7qjj z5gkg49vmV6E|*e`%UjxA%9&?COn|EG(b@G`3@!m+5`&)ZVIt9shaO)81kEV7Uf@8B zZ6^1FjC^BMCd`;#_l*{bKQc+Cq?5O~39s9`c4KLGLL2H6u_@}wrb_jPviwE}9Kj>0 z8xJHnNK1C}8z$jYD?2b|n@L2i3QqAzO%38vD_*J;!yKuCiJyYe z(PyRTwv$T=X{IuaU)Xc1Y6?VLho59&?P}$!i40W8Nw1+T3}5`KfEl}g>!oMqNGeK$ zhV)gx?kdc|ujBpF!lCu-YKXW+PmOOoxs}-G={p7b)S+7R%{M>jyUy3meW`NmF{=95 zUjX$l;Q4~Xa(vgV*ZE^A5n$T(Of3>6g7s8B^e9uHLkUwA^q8_*kK;=yjFO9=_KbKJc3vI$P`{ph*(WfeKInOIQ!yIPvkL^B4}4h(b;{OC9T z8?WqPQ&UUH7c&BcFgCVn=Q`GYln8OM-c`3+vT3|QZcGF?G18oE+)&PRg|c!oHEPHu zKg5w=L3OK;-jfz2tmdYkk_HG6&yN}#I)o@CGej}s1;7cY9@1Pb!X^96iGv_fpb{KE zWyqWn-5@zs*4FRH521FBz|7h2OOWO!*yJVx-S7f&mLr$8b#`J>%os`No_Mq`joRa0 zje#=#+qXYcupsFy-_tp~2a|W@G(?EdrIM^V?O*ro3Mc+SVetk9Y;$K6i}m~ixExbS zSk#mAAB!Qh_JZI%CC^VhCtXG}cU7^9+SNkmV@V;ODgwpHrcec+t+)3GdzAm0vhJB# z+%hr=xUMq@3(pL_Wte{i{ASm^VKAhooayqC1+3gys#4+mkA^hBif)!TSKwON4{p6G z{UpL8C6|Pm|4J6XlXfaPu;N;VemUa@!H0;pwrY8{*7Bj5Fd$hXKts#=h{E?KSkmCc z))9oz)DM#|)RU}If)G^rhn~fHesGJbGz{1#U;ON#E}ZLsjj*5-$_ZBzFEXAg?N1Wa zM}sZ~3DGS5o#GgN~6clMExK%h@0a6uZ>>FO1pZ^r!=fd0tyk1$+9KM z1{o6r9L5uliEYD@I)FU)P)ccXJyvNpH?fgI?@ss*YIHnvq>wtzR_sNn>DjhhSjb0Y zBZFr$mY-sDnk>xBDLlQ5qJjSBXe211vStqvBI;N$uU~f^Z4|Y4H0tWJkCR;52(s@M z$pK%E`|4=;^Pq|v{aeD;_q03BB|19d||ciYh8)Hh?#4 zY-rRV{5_o+9U}3#lcr+a9{i)FqNAbkFD^m|eYy3EzJa}nzJ-MaH`+(gpdk&zSP3nH z33K|mT})-S&+{h`oMGMm?JwNWIC0@sgMzB5of<5xFN+mlq93d5mBhqU7f**7#vmgj zmbUh%WlQO~0;;;#h#=S7n1H=9x?E!B38ufz+G8pyMQFVsf>b%9C0{%ypKALx zyQa6G_BU<^M}V-fJnF~5t~-=}p1DKNnBO(yrBuTA1AZL>gI{d1xpPK6iO_<5eGS9H z`qFnjtst7R8k&u~g-`yJEwgi{_I9=(GD2yN|Ng+j+-Y)mX=G;ywB>PT^(Y=T@@%v3 z#rGA0;+TD>%dL&xRi4oIH9tr9Stlsqu-SEgJ=-k-wiL;JcHb=-oWLj zL6^>;C~;VI>BavFjqnt~kU|b~ihTE7iuZ=^>4xD*IHP-boxif8h`ysp&YUq%F>VsO z7+lcia-S4!((h85?iOZZ6J0%>4dn%8Wi`Xvv~a{=%gD$YTRlB!Y%@fJ>nOx)+>E=& zSL7JRoY<=9EZbE2n0+3Akojd*Ei|dTVpyjo7lm-5ju`fMX0POH;yigePCGU_^4HSZ z%C3C1nndqn#CQ)k_CK)Kb5buCwZKPmt7% zo>l8oIsqhtCIl@X4Pd7bLi%>iBqU%w8Aj*lI6~TE-P(QeUP(XQlK_`#ZE5S+?#oNq z=h&e*go(9x=*m8H$t$^k3v+~@akwZ(0DPipE4?;W=`B&dA_hwFmq8nR~T~Es0=;wDI!)GMwG5+ugOuVBAxt(e*5ZV#dTY z+y3YQ1)5|(z5a-Yzf#HRtnXJmXWq--veq^Dgp#s*jmV}TDKK=|OP*zf(=9YRy%>Nt zP}f1fX6~$6Ru1e7788+48L&op5^J2RoPyU;P9A1vhnVmvem40bOZS0gIy9a$QijSZ z;N!NMk;-Zbw6H9-DQ#~?px$5A#&HqpeM*X8{~fy8TIu$0xXOVYUmhQRKgdbQXw>b( z3kRc0aKO}38drTIM}JFFPqkbGuq?x<_P9AKB@P0VLefh=v;1zt$ge(FB|g87G|S0j z(qA8Nt8TK}@9mbGoKOB5ftX#C!Bm3X*7se|0G6=jqz5fl%me7*;T0-s01338%>(|M zw^+J5(ui+kgFmuO&Wb4^L6%_OJY>|2i{!PW%aXG{lNHNpvKRT&M#2SJ*+=3sk{7ek}^XlRGti@{ff)u5l13iN1ciAK4Ik)q1 zz2tt~t*>-bOf#j@#?tGu(H**n0?w?3J@(yTpB%^DO9FgjA` zjhC-fYCDP0^HiAq?C~w8gs1PDhW>L)XXlDBi+j7kl2+C8dFQ+-Yl!IPYSx%Ulm__a z5LGMvkRL)U_>Irf@G%Hgm&>R(d^JlxX&5c0?v!w?-Hq_+VVn5Ndj}O4*Ni=|aS#p^ z6LWg4nBlZ+HPgN(7&UM+!MDNiu$7uX4!7=e4bvBTJ31ooeL0|{?|X~b7$Tl#l-3t! z1DK?|UXk}b1l}wuj->04Z^!DDz>}-9hmOxQ8C|bcQNU6l1I=VRPF&CEP9m_K-F0YV zJNU&4Lm}Hp2(03%GAr>C0-I6g=NQHlW9TzVp_s`bFf5acz9-X zNBB)a=N*l7_4$c)H5qkbgydVY5Y!sbBb{w@xY1eq_q~*G_^!%PH)6yQa0z2r#CsD4 z%fzAwUXHHgl3)-KoM;g~T^oY_7*2YZwX0Vy(d4bZeqx-A>!3K>!Pi-FETv-4HZ`Lm|3zZ72Mix@KQQD@$N& zZFlakyfA!i9`%)jv(ckd@Hv+~lL+LIqokyS_%RS!VVtZ@i}6d4{?N5%wMDG@{0%LW z*3F!E(Le~WWo;FpKu#W=6lcDyx&|8i}n+}3l3b`jDq0)<3X`= z-6dsIy3p`#&JTP}g@|3-1>pZ!H)gpW$!tATtekqDR>rZiP2*>1J2}_+JzaJB`BFM3 z%1fHqyUq7I7R12z5pr!@TI?l6ewJT-{N%6mJWfF7QFBx&-;~(J@dp=CSowudW z0yCQ!>>WO9%?6-1Km<>>28&ZpD#ohp-94?Q(o6btz(mzxpjUi--kdSIti1y2YjE#p zB9jc^6#M3qio%0TrS&>3MpG=5_S3{`2&R(I`%{>}Gip;~gJLe{Bk zlFIs;T&I_0Jq)if(K-DwWj8l1jKgCw1-x7Px?VvLY-T)U{ao;*f8;0=uhb+h6JxAo zf%~eHf}1(4iW--xVK>t{c=rnb%;UZQX18cS;7Y;0*<1KskgY`Fi=x;j5dZek)z@Z- z_0HjTK8h21AzS`M8;KmYV7E$_kFPL=$=^`JmYO}R`N3P(>_u4XEaY5u>HW@ z^y!lr+?_$^Q;E#BR8j;0G~?F5I>5)aaptpV%Ssk(nPG2p4j(FtDCluJrn;XvB3VEx zu*L)U^zf_N4zlDNbRV{>fy1nVIX69jR-@JZdNQuk2bc6jmBd2L=e*UT0w|QRI`-iA z7>m7$3|TsoV8wJURsPqLa^jb_wL`0kpfEZm!NXvPkjF5I9=ocxuCg`|>RYYjX6UEWyZ8S&5yAWzvxt?G&N0Vu~FrK&-htvcFV^74E)`B;s`3+ z9|DsMe0QPh6oW(@8Z+b~uN8BIZrAR+IFBvbyA{ zQplJrK8Kh8fVJO2*8(zZ=Lxje)GoEaOT0kUa}iW|qL_(ajnG`N#kt{#$YTEmufc?@^M>hOvS?5X)S=(KGd6e(uNXcqt>xbTqoa_*KT;9+!*s1^9_hkN6a7@qhX z=NlmFp0qd!a34fKpbUhMLB?l*9%pCpYC>#`;>V5Nb2VqfVpLE;zm~pQj$xtD{#193 zkDm$UU;JB{o8crD7UVl33!om~Z}xJ3Hym16`RLj93W0IFZH3QVKAYxEbg|LGIsPyYO~O*FNs#ve7eAmH~KdG9!Xy zad~`dqF2OQ?;;VY6mZt&*P;A@yE1g_{oZH++_1yv2@3!;I42JFXygTfMOp_8Iwf z`eNiMDX9&tPifuHqS@w5*L6Gmtt99uCnv`@>Ii~Yp}|!2X{hept8nP{Ojsr66GT>= zCWtIv=3B+t=2}nuv670IqU*FN2Cu>{A#nBJ%4(manTG}<%gJF$=QraXwky>28}|X` zmzcV6(Q$?BT`U-4zsJ1j6=_`Fj+Z%ihnZvCYChnNFtH-U2UekiF&;IwG{`eU8nvAsP|y!f#w}u-~=_wIVdkDP<$9BXp%p zBdS?qIv-`#pnJ_%jh0qw`#A@nDU8yeF!JpI+fY~4@o`9$6jIDuIHrW}bne_NhpBlXqK*kOlJ7p_g1*yfCBx_xdkD*fw9-nKjf1O#-lqU6hSAg2 z`VHgS@Ik0wF`4eVpHW5O>wroS6hAaGx>BFDMx#JKl z&{@@)1yQ$>tqBR5&r5o$dgV&;p)C03O(zT~n(?*$YRfsLN*5nNiuunexILr75~vym zzRiD)nicSb_I5BVZr&Uhmv)KxtZ`Z~9XFM_zv?yw|4P4Mmja}^R*SKFFJDkGfI7Zl z?;%Esiths6WG`R8@};qZGyuuWZWFS3uJ+x(>8WY98J~9R`<4`xKi?!-Z1|~;?a%B% zasVt|rR`mUhoEu}$&NhE0cZ!R^Py6*f0qqyT~KCG;M7nE1~dlz`n=jOO#2-$qr zPY#Wr07n1yImK?NmhbVIR5Qg)o{Ckl&3et;|KH&F%0Tkf^w#k9kl)+w{qDRb66CPv zd<|>~_^h{<^Ln74aL4!J7uZU{A(oE?kZ#()5Uj5=c~Gx_o`n5AkPi5>knX9yziZ(4JaZqF0>JI*Y2ysRJ;JirfPYHe z3>)tM00QFd(IT5najk&H8W5Yy7wjdqN}H-SN8~01jvK^-hxbu^UIv<7Zxych=NF-ew(*g?;Z!7D!=O5&T=80eizES{{pb$gf8xHT&O{qJNsGg zcO6o>K1;T&+Le3W_SD6mpo)za-_1BV1fgVgyDZAdUJ?6U zc-8$a`>vy2vHESQdS`6LB63Ak&vD?8UfurjG1YZIq80rfgn+n9zw}wO4uU4}OPP6k zb@uHIE-scxKv~|P-EC;Y!S5>{U$)tjh}e_75Gjk4@nLl)SgRwTnAReLLrBQ`?ROG% zK#g(gUfS$jQ-y9*XJw`LYH-gsHPvKCKG?`lMBQ75OG4$>kDGJ%Y?FHG-bbl8b_VU1 z9$S_ipQplT`LK9yQ_Ur_g{+4cp>CRElaxV25QsUnk>34$T^|=gl1dSm)y>%W^lo5b zO#AX!(Y<8Dnk!O)jDWAu~MFQQW4WWr`t@hoCPgo4f(-8#=b)DJiOez^B@SE#*dV{HmZ_aL#A8nT z*OO!7E2p~lwl%R(6y8jaHeDB>B7k3rcW8RgLIDlM&c+XYnwo&CtPr39%iLDeKGzph zSM@+f)gjr=R2MinPpy{sHV)m+Kry(P?vrNEP#mK`tbcequC^=v{ZUMC5}p|hAy={} zEV(5V+={R;XgwXYap*ek0Si3luZx{)VOoHb(af0tu(!n4JtMh{w*T)QKxVV%hWxySZAvx&b z_*_VcQP3p{txlU!`wz9Ol*K*3l@%-GkQ7(wH-AFn(Q!Kw)Yekf|AH7dp3QT64YLuE zOIt8Cxj5NibnW(e?df~6nMJF0+uD|R^L$fm5&eRm$HZapN*FNHq^5pZINkVKM+hR| z{nqNe0c8}Z0ptc2sydrD?&ID&UXsUTGUvPi^C(2Xdfn4_`F*uS>r#IqJMx) zU?f0HN=e}TD5(AZ!zk2->&1akj@QFimIb3`BmW=V#?3lRyyxt7|5{VmZngslwyI4-J~0$>QI$fN=Y{_`KtRNj#=;HHa3Q z2I6x%pHdz2rw(+Ogh)*laJZ(8Il0)Ia#+uUw+E3hI$vFvmY1jvLd8B$9+J$uF8I0& zJU$R!b;Ubn{;+LKqo+SsuUsdK^+Um19NKjPWT_Ye9?O5A;krc9AbP+j>3+_m8=pH< zZSXs-%XrQ_WqXCMjj6>@U&OB|?X4$lL(gw6h#)K=rlRXIUqio5X^i*zWJ&0~E_VC- z*>qdm-uLn*bqy{UvCZkN4gv32oEXJ#P$t|irFZiyU-|bt`gKzaq>iiFwj6^*auYTy z%IzFmvvy$f7d9^XfA#S>=|R(E^zy&oBoP-D@mwL+XW;Xha^EfubqoH0;6oYEU`A$) zg@#1s;_~pinPf6jmCr4%_Pe@x3`QRdIV&35jd!hm$IiL5^!4R~^eq@vJnbCAoyOwK zjq~|#rRG-MJO^EdxrpjN><3y9Kd+jzITH9iraKt;o$s}l^xGB}tqc+iUCg+r>YUb< zRtl??$sv&Rn9}^Jc(_bUyP^p}{oT_K%C|*G6&9UV#VwMt2qJCGt)sQSO1Hl1(F#2J z_%5NkWrb9^Q=KuXNkFHbs;NY`obZTNt@gGBH|zyNGMgMR&4H0PnotwuH)H?C+XpUd@fo%1jYexe zIXBxb$vENrx146_Cf-Y4wW;lPuqk!z&*KB5r`BrTB6Ls!?~G~BF<%g_ZcaLP8lB=uT4|vF+>AZd|{jXy2M}y%WfQ4 zGRw=3Ic?YNS@GoE=nq`}T>qH_dCX<>Q->Q_N)DC6rmI&ip;_895YKj>hj0GDNvY73 z+cVbp%~Qt=j<0U_27FMrj#O1;zgOB!+|@<=#C(&ut)M`AV`<;kOJ+|m2N6Fg&$h+r zNQC4EzI3eGGUN5{76G=km2woq%Cax6Pa4V4KLB46mMU-+Q;V}?!6__*l#~Mw-Pf1D z_YKz86IzC(aVM)Gy^yzS2zYg4aTXn=yrv@MWU=x7UsJY(@5mDheg%X^r_+r*o&KhB zWt(ue8{m;Xp+8;BRD*SEYFsYh1!Y2hd(8;1qOky}W?UQYi=mLeD_y_KetD|vc+KbNCS z^u50pb7Vbvv=qY_x9;PF>_1vv{Kj7Y5DXiK!*6M5Fp@Dfb(rEFJcvwMA#6}f=Q!|u z%Sg*Od~R)p&)Y&QK?=Hf{Lss}CRt!~5bw8E$+&9<0|Ub%?(FS@&w1v=Z7W&ech#Se zPnauoG3I=3W?mSF%V9Y=`F7rQe>!azF+t2{{zJy9YlAq^^9xcaMBu(kj>-3;bzjiu zbPlF@?QV52roFz7Lgnu0Xs~`F;}SycfUde~e?EXMu;ciAeO<4aGJ}5u2mllt;D?@V z0R8{!DFiGNCj_ZWwg@4;G@RTt39dG7uI|sz$TYYtWn_3}3svis<^x?%v^)1OQn6=B zDPFq+4%#~HQn80zM=gGtoCF^C;}XW|ejNuLS0&XLsf6Z%+-l4W%Q- z?LD7=N|>JI85Q*!s`(@{3NmM~uTxbyC&%QA+J>{4i>&&k;Dp62B;pKxl<8#IHr1j) zf(dkyRwEFgV3ZpYi@20Et1|8QR*{gnLnjZFxPwlf+5dZwS%NlMCkoQ6*!~X-!%W#==l19o(;G$u5`QWX!kJD_#lps}VZcU$ z)u>uZOU)Itw=reLK?=Qyap&i@TE$5*iGkgoKW)#@Sq8}N;kfcxqbwBQ`!hP4|E=T} z%k$d{oZiDxX3rqV0bbMnX)WBSO_!oZd9$BU$^RRC#Src z2s2o9x+e!hat(#dcxEg^392Eoa-)I|H}vgm2q1olRQ^=HM`>a{O-3=)gO4%snjy2= z!sNlDMCP=*5mzw%`n=Xs)XY0*gujY=B*565k3fb)TK)UiA{4}?3z*tdT@s)uI zW;1&dHv8lhF*ZVVUO-aZhT=xB$o`Av$H<-4~&76s23rJALct5`?WuvrT)DIjsIIOjntek#^-h_Ew#7wj_FOd{jC&88jy3whZM<)8{Z-vP34#qTmkMtWF)Wd9JP z*hH9pq4<4s`;SxBk|_hd!q`X#hmE!VX0vFRRDnuC4~?R+xeJmgCeBR}WrSnIa@*k1 z(Gy@FF?tM)ol0k%cAu9qW==GTt{wf^a&j$**{L4{0(VD>|FwI4A`FY01*DAh@xiUF{xGl66kj_#J>Uj13<0~zBa55u zSKNDV7Aam&Y&DET+*D06PrO3`}B-Y}cZePdaR~4-s=`D>Twm3xbSe=_;EA**_2O-f59KOt6mZ@sfhaM%?~w>B*%r zu#6%}6d{EAegE`*p=AEEAi}^U!Akrc_kB_*n*Y<_6Po~!g5&RN;>CUP{h4Z&(^Mnkqo> zVZ(yN9=-WAzh|OGTngsr9oRWo`|j4<2NwnJ+F8fvoHFV7h3tAqm^|GuUq8YbvFhLK zr*FaZn_F1uGUIM%E&&<&%O^KYTNRT$#qtyCHtW&zOo}$3AvAQbh^n;;nDqx)zXLP$ ze26eKV)F>&!q|tR%EBvrLl94<)N8^`)s6|S&OQVLk`qVYSfT}2;>SfSw%$FH`dc2~ z=O?bM0I5W;y^B7eN3_bS5fu^0M8!)Rz ztqP9JmLXztyz!s2750jP<7;b0*!Jr6U_fToJcOMc5O&ng4Fo}q7{s<&s{-F`EHOkl zA)lXr!>p4H{hqc(W4dP`bJ=$$yk$!%6hWfZ>D*MW%saq&8s~;0F zV&La-bW>UP^DJ!qVfdw0=cLb`)nuA|r{zGMu8fMPoLr|gZRg$T$NJMRP8s{W!Sr(9 zQ5X;~=on(ZoQV})04GB`^v-3iUqfH~8K2xbzbL(4hvOOff4drJD4)#tpE|ytFJ=AS zd2of&qx&;s-Togwyjd3XEIAg)$n)wj=WaMSYnf-2Q6(V~pf|7Cdt|qiw!8)p8{+d= z!t+TeJ@5*;Y%$*pKfRnd2IgdETc~f9Q&A9)XUfxD74!QY{kQ$~VA1|px;SkAwIb*lviN0cKeKH)}o!|+t$u8!rXjF5W?8-6%gup{&N+?VYO|_W*ZoK zDS;X@L=^h>Bo60l%h|SM#!A8sUvhq}iXV`98`?GhOB51hUan720dA+f1CA?$u~n1C z-#Y$cC`mkZ` z@e;xTG;i^N$3{xE8?BwKwJgFscfMJu|5>rTeFPdwj<)CUvSbwX+Kw}yo1c#K9K#MD z=b*p1PIxd#gr6U2NfW#IM;)4aUQUYvl*zr&l{x`t-yGQa$jG@UF`^9jk9UdJ`B|V| z^IyC`bSQn+Sa&5-;aKJBXQlkMKcQN_i*Ho$g^5c%wpr$se=YFxs3lj;Ic%TyMB$zm z7kQ97>)260T|F<_uhJChN#t`tBkxFf<@uW8H5W+v_37cKk*}{U74vhn*?L^>`}#G| zVTItgd8aH}coA+fq$n9RYu<04^T}R@uQ9e<$I-M5nEs-YF4q-xxee#}cZ;_@D3pc& zmt`eyZd^UH%g(|yvet6=U&gMI|KEmAMPYC5Oxre{_t%L~?{=Mg;X6M9OwyCtNiY8_ zva`ijbRdt#A+r`glGwXA8O<3Fqk=ts4G5E1vo$3p#zz;yIL<~wj(*K0Jz=XP@6x=S0B<;BhBQTTp2b?c6#aah@mM_;0_cuPT4SrdpBj_rz+Hi73 zNGlNX{4H~SSdpOUU^#YH;+RGiMaP*6W>{%_@ypkzPbCuMC_#Jn>WL$9dS8L86{|V) z&ivB|gg1(-GL1c7yCMu7X(#1BO3Xf0Q#$EgKht6_ktjB?IUEz61IhIQKZktcnxG#l zrN{(TnAx;)b4XlQph97s4Qb3sxHr`hFr}6Q@|vzBrOKti2e>1*iH079qx=nBXAnR!4@u=YEfeZRYyd*Qht=Hr8T5~UC2=TXofbegg?xqw;uZ=lc zTim0Lff(jzD*x%X&{|H~NOsUcLIdDsx@Tsssj=mCetRq)P7pe5_CCyakCEO6Ii195 zvFP70@~P4w4&h&SJ+#y=9j}wt4WCSM{se(w)`4SscI&^1@kz=rSRQNqe$5QMbiEbr zw;B3p-80xEhzJ6m9fK*H<^ARyjZEe9$6&E-4t5 z{be~RbXN|HPF5qNYXH~^9qnzmy3Y%O9DuqTFV8jWM{mG9x1(p9OJ+T};PeT8*$+Ch z!jQDZR-(Ed;2V1Q+q)@DKdXod_tVlmB`TUqpzJ*M#>N?q8ZmnIg(hPx`|veF?xsW+ zO5fpiWWPzMp^>KU@8#bvLpoaVf)0BhzD)|BS0@kN1Ni+K9>dGs>GAth0f;ZeEY>>@ zFaMQOAySN@j_2|%v~BZwCzVH-iIoxRM+`zm@b~wZ(@d-CF0QH)!0eOtl|z7bka0gi8Ud4CA<~W8GGjG zy7!Ob{xAMMs3VFQ>~~wN8v%p+2|#7{ePQG}2!=O}k$Dmb!V910JP*eba5}yXs43-V zWoIKU8F(+eQ1{vyc9?&c^o;f+_k>fSBaVL(jK0X2G-20^HOLJpMuHKc!*0=@H2*nu zjzS^A@R>|`hb+DyA#`(F9{$>I6NqHA@J%vMaE2ty6u?-s$j~yt@X^nJG8z5r=zfGM z7I_fV5BbZXnvQ)R4z#8FX!xldi77eqi4CHOPgfiZqSPMerN85eFNb;+{$=}NB}=lq z_a^NV3NL_kNG%qMh4Jpve1nG_Q$#r{ZR`8($J*Cp_hSNbQ3F9ALe7bvnR1X+)pQSpayDwJIvOJrJeoYQ8qQ6*tO0dWbsju;)Bc2+a3OTLz zf5zBq_}e3u_p;QIZuBs*+feEX$oLacg+yU|r%(`umw$kXpsq;KUHsJ3kfrliG#2dT z#w*_;4NZ@oALNZ$C|CP$FH+!6u(y&g?xvw zO&-h13!Qk0S*GF6kPc|&o%;^DF|;z{`M#vAlUR*7mWy&e#_v+y_OkKNB|&PWuPBRC`H9e}jUFJj)9BA_RQ zN6E|_U6sI&4*zxdiDtf}SEU{}|5iB0O6puL9gLWS_L2Q3SVquMTG}1hdsP_v>Hfc@ zvA-g0=Iq&kI{2Am4$$m?Z+XwzIfl^Hbb4*~&PzGz_Zs7c6L)~RpxmyMFC#!HP*$KoY(4btdUA*GXG0@3R-^NYBu3Wsv>%X2S_^b=TBd zS5{ZDGBSgL##w~8cz7Zl7`5!t5x$~+g)2Bno61@(i|J|5-iX?Mo&TJ3mYcVp6YIqL z>-y!wBs$%`xy5~9sV!rIrq}4TW)WV=nBW-!w(z7b@NXoU;n6%T?yTS~FbCVc=6>VZ z=KYNtwO<(<8G(vIiJXyixph%t)mDhF$VYFB(R9Fz0iWZ3yQb5AgX&4F}ppO4N8MX5V;gE(MtbL7Y_lt zY^2YpY$7?wKk#=+^zWTGWvKn)lu2W7k=a>#m=Z{?gl{`y^Vq}AQs`RTTrKfLUv>&3 z|Gm8QFQxgvkNzD*(PVitUD#pOWPN)XuG{pzdmIMjnl*m@_w=}50InQYYZd69XX`(E z-`6aup?*R{AVFU{ux?Uxn%Sq2Ke*4sicbnY+FDY|-Y;v@+qNYyJ3-afF-uk$w ztmE6e^H%MbX9+IAsgDmw9zqjbRsqPf5vw+g37RdfEhlb(ZTS#^((d|3n2c0-m$S8) z893&J^hF7S@1B8xvw&M7<2;V)TJ&x&H6lfp<)Xu~Jk6%-Ga9h5ZT4IrO-)iI{Qc%E;yf9svM*3$l4Q_m#VGc|X1M=a!o$mES4Ndj`Y-Z%OE-cet8ary~TTnbna z&3cpOX(sutI!9~Uy2{$B%E~W3M^eDp&Vs_)RicKz8}px5}_hUpj%UatR3&asq-z0gm4t>0%o zhpi_gnA^ZtEaK}ns+NyvfZ|uP>uoqGKhip(FO&1?_cyy3r|Zef^UJ0jNU2cA?P6=6 zy?W7}gN}ZwD!ny+wgiGKTtubBrl}mylP^lffip`BK+*9UTriCXjE236X`E~PCCv-i zBY`EA$P z+oFt4`{y@6kbRhj+h1aull}-AnVz~Z8 z*>Gzn;K3<(8c$x%lasSm_~xj}_f9=VLjGXS?B4)+o;nlm?p+PkXmDwrZOfef-*SN4 zNm@T#B({-{g(;}JrPTMlmN45d0Rt3Nw8PXZ?DwEN|Mtb5>3iMlvml4(_z)Fpe-uoK zHX<09eTdm#9o=AkJf{-0JmMZ3u=Dvo`nF_Fn=nZ_T1zxmI7z@Q2rIN?orXCc1jUnb zURSNA;oQMah2ev=eu^ySGnQQI2ZTOCvG;jqZvcA;@Io*){KwO`n4cay^Q%`f`Brqk z{L8(hMHZvOM&{va%k>`W@^$PKT^G@&*)OG@r%A|Xcb_|*XT5q39lhszJx5M(2Aa5i zxxV?Y_e2TnzEz35z!eR+lAnCvAAd|2yy>-`>;ANg#!qek3%wqS#edpRlFQQ7Me{M} znAcHH34D6t(`!2yI)R=5-g^a$Nx)@G`L7NEgt7kE20Sy&pJkd>I1#|wTm30M@MKGg zMW@Gd{LVmw>(-tmx6k+jz~n_@Idbu;>h$BI;_m_iU*iLcl91Qj>G8I7iKF+ZaGrmu z#}+W1<5|WpA$-F@516V4{9t_&|E17r*`&$Bq5}Zy@w(6bWOVF!23#+EcRhj#qf=84 z-lJoaYaW2BNnV=ownS$UwVcTwouDjr`U&*L6t)cjq0!F-0!}Rp`X% z&v5q}cmM59TbP#^xjqVqS6vTBPm_%>k5H8}{9EpJoMmOS%8DOIKQfK&JgE#cY;=+K ztKyC@u~f+KvVNsW$)z9YXPd5B`?(*|q?EL>)qPK1vji8kVfn@B!Aea|&*3G`Cd6*F zqqwZLki6%Pz-)bGh0AUvsME3g-#@)NpjPB_E2%84+1R}V0>OmiTWq(u?^uP;v=yPmE6{pTyLEvIsXbksLJ|6E{v zPlt_wd42M`IB-4Tv0tmVkkZhw2TD0Me17Y@W)pG0+KyJk@_C8@t~TjkL({^}(_JfS zKrUIwdRA?^TV-pe`C%#b8H1E$hSkFeXi|RxxDFj|`yI6o34?YXr2@|>c=^s_r#zdF zr;EOCncK2@?UwVk-5(=|jJ5?i=I#*h@x|FIeGuI+X8&) zu-OTzi)%YQ$bc4(LxSV;6kK_pA8cle^Yy>(=LE~2IU(T_vCZc_M$Q{thk(@$33
    !wKpfj>*Duk zo`uDhx2ti4$bN3;otZy8Jzao2{eR2m)d7rx~cG9{)7xTa3Ql+xTp7y*2 zMOV)5+t!pwgsuNf*{2pNyNxyXeU(g!(Gh6Vu-EmZqZ-eP5|-IDA&;{R9-8XdAz>vU zrADTSU>y}dMx=qJLlcZ;X=6see`?@N^!-7~rWRLP(ZLLC>tYC(?g-I{nSwE8-1rOx zY^ik&64I-Zh43*SZA<41c9Xavu1XUh<__OUIv!tOzOT3e`>dh&XuD(J#Utx z>dct<@n6^D@Zfmo25HmiL&7)1ECJWb2h}M+RzuTb`?2Bh0hM?d4;wJS4Lt+=CG4RZ zcnS|m>4}Iktot8nW?fDS-{iA7znY02Jt?dKEz#uMOibiRAfGV1x|xJShSj1wvEx>X zsztbbI-XV=u)Was9`K4IRy1nfb+kS|nCm4mdxD>45x6Fo=tFOrT+v>7_BNdN{Fsd~ zFkn5Lkni#0m3aUVwlat1u9}8+YF1iqg^M9umi9k6YVQ08Jn+f! zGGH<0nzL{D9;#%p{+iXBU^IjnhNr~1tH;kkZ%Vo+#td*Lz5CprIYB5V;9FMgv)j&) zBdPX~g%XB+|2UO`q3qcTfSC<&_oWoB0o~darwm*p|6njEM$+_mMtZiwR9?y`AbRHe z9t&scWbSE-4wK{mhIPa@(^pQs&%YNIc3!*z*ei$?fW6Y!py+$`bbsGq(0+wuB`}2tyrSm3TX4-sgy-$gG|4Ilt+Dl(&1AA_c zec!RZE9LVCJgGmDSaA?~ z`zq0hJmqtt%hn$6!DIbWwx`^T<1U>Zp zHtDqR@xvqSXs@I2tKTKh7BfIxneB;)dSOttGRYf;LrCd)e@pOx#@+C9-D%+mEKyrB zw3{8K`<4-fAmh1`!ekmWjT+;__a~BI&Xi1uoAI3+rS^k zqx#0e{xS0>DRiEm+tr;|CR)fHTk=STGmhC%MKyci>tKFYkFrk$+;vxM4yXiN7U5=`SHr))g?DcRinT9%dvWfK$D zs%0e#t7RsZ&HWS6$r<)KPG!YJPlkRx0s;aka{-)9fEz&hog-MZ;={t4u?mCHl2+G8{E6Qijq?$+0qc~KE`lVBj>bYnhqvc|j^E~nZ3 z*oW7)@&22uO~`A386&pi@7;K9?{f^%`RmoQ(I+Ts|3LihlR(riMBtG=W*|fOGvcB7 zw-xz=gdx9l>6*VfLjRI^x`m8PMBnoKTkworBh|Vpb;Sym!%TF-L~1m@5!|#1U)+=% zz0s`%KVL4N2fnzw%u!BU;yIYc7t^VFMMUYA!i&I*UP#53700uzeO2lvMvdSkboWTx zo+*6t6e6T$jOLS16gP3+KGNF4TO~J2cM2m%jNgX{Ag9WPvmq{^UAD0!D+#GFq;g6) z^Wx6!^VU{MJvLfY)dH{&2ai7v>y5il0%UzbXg!YS!(HG&8CSYo(oHYn@Hf( ze;(Bmv-C(y8ny=@N9RY)m-01|HOey?yV_}L^r>PP!J5S|Eb)fCiY}l=*-~h?BROH5 z&F{)ihfc*2{Z12&pz`|>G~IR{6xR~>azfx`W%1j)Zr_jxMHQ)CtA5HSG z7nn?e*-z$YN5JHhC1C1@WX%{ip$!IC*Zn9@BNX2(zF*_Zc=?3>!+*PVV`C6H?D&gL zVjm3+`P`PH$0ea8oVD)5f{kt2E>%wIhk6JYH1LH91cFFBz6Aem(HrzZcXw}3kzKX= zO~SvvzG#V4R=ZZ;Z#+ZY0c)ne(&YB5uE{9ib=;wn&G%pBH_5=fTzmByY{zKTKZnJ1 z&?4yxYGrO%)F=@#0_fhf7X<^O+@|-pmm-N#mU%066udY-vw@2=+*uVTEEiru*6xe$ zEi!cm9OO#{W_7=ADugh0+&7LwI%pYPJRJHibgt`5C((KPt64AWS=Le+!b$&q?6nu% z5V8MXPn=OU*QlKWkZ=4iePW?3TJPK z?k@~gn1JP|qU55}AadN|y~kg4<~B!aRwgeD8quv`mqlLe6kutDFVtwM>$a3^NCRVo zgd*lOGrvj@!0OBanrv7rV2swG576`+EO0{G5L_K19Ay(qf_-Yg5lvX4VYyysm*r^I zt(Jk1Y~4~E@9jY3d1p;?QCCCh*`I&@EOkaz^E2-c4wG);Dk&jl?>QVNZP98r-1cjp zKemBCi1cu@5^Qd(DBV^5dIDW(n&Nxcf2$tf@Wpsxd&5u|Gbw+^jN!0Lo7L7-ZGJa4 zP;*@?d@3ttv=uZYOHgDRA`A_fe$y?U#YYYY5ZT6_yg)Gephh6H&`Cw5?_YPzr+|D? z3;a-kTxo^f3=6JjI&o|FJmb3xrz zz>l%OHJR0GzCEWwJZwR5i-@UKKDF6;7^GRuedKP_60<~r7@@0h2K<*l$G=FRWpjOa zL4X#~2;Eujzr979JhMF6!C4?N;NDh!Y7PsEmK-0KMMHRvH+b%_iw8g^T6Jq-h?Fbs zh!%(Dj_W@yt|lIlv!+JJ=KgYa`OckP{#);U-(~IbJzZIXDd=j>5U!*dYh2M8n%gOQ zZHXkxTFuJR!(*-U_^B`DF;59#Y_zI%eg7!G0vL0wlY8LR`&~S?-G8|YF|1Rg*HD-& z1D^H30j=a3EO?2XS z$K>ShmKI3_*JmtTnRHuy?m|j+TO{%8maKk9o@Aph@kkV-N~jRF=vv83mD>ZfOhf)y z`(Vj}4KSStp?mm*?0rqWsgz1!9AGazOq+3zHD^wIK4fh_={wY{)?cgAQi4J_30o>Q zIs)cBK!1eMqX}EGn0l2mU7ug4TQiwbzl;&2eF%$1*ogUiyXN_Ce3XMy1mF;@&bt5X z@xFVp_@2IC9IsZPt|QqGBuWk^j3nn%`U>Qlzj6|uow@)gr;|gYa;4n%C`y^EGHK}V z<7N~*EytDCa&lHJ#~D-fE}?e-`T|&&A1h|hzkMT+hW_}nyu7^LYI-YKKK^M-pEW00 z;xnfVmSTE&^Us2oNtYw1rupMcL3b5bxlY6KQhES6R-uj@r`zH?O%XZYZLBic70!jw zEz~nX00WXp^6B;%okOeAbey%(kSsQ=+7|x3ynSQ0(((R^5g71Ze5oVp?@*g$!M@cc zA<_Qmz2bYns8ikA{R}WJHwd;ku(F*AA{ZIQxCq^R?|!e@|05B=uwQS+0MA)-F^$*i zG<7yt|7Yn*#=op;v{g3yLu~t~I&<%(QhfujOD?YA9ospMfEU zN=33t0}pq~Y zp!W8NSKsV!liHCWf+jb@nwVZ6Mld3o(C@c3Rz@SjFMWp?si}w8y+?kV$`*A+a&^3X zZ(Ko}0q$2Je0t*ySJ=o|{mZ-P4l7O%EwjCKhCOXk8lo=)e@e{gH%-|0AQoc8PK5(^ z^Chgz6_B(nV_CJ`S+$0#pCWcL8NW-!E0E9tc=#5QbhV%%flwVpPTyW$WFV3^UR*30 zKAT8+A{;w3lzc2z8B2{|xLxK;G_u(Gr__l&oAwMPNQ`9B6qGKZoDx<8O@U?np%aNh z94x+3*s=cd;;j~_m~%1Oc$)J;j`oeu*5dXlsJ~)*dVGBHz%%aEthJO%96T#sQaq3A z(Jtq6+jSOE<(rbOm?KzRv9NdNb)Y~noj=L$d==YC=2cedxxZxsr=aGLuw-*44nO*C z$iPfL_d_F^FB&rH#B0Z)$A|t2eg|K&dC6jAI7)v6Hmr7MfiEP|}KPx=~7 z+d!!|IXw# z=;6_x%Mwz?m!?sf0=xpm#0xL%05(tH*PX=Pr4nDpA?Z9|O}hE{UYHc$T{HuQS*nC)DE4N8eFU zuo~R6%7_&`wZ7qrssL0mVhI)Cy>#k17 z^XluX>KkgN9zeMK8<4eP@^LUDW%7 zE0Vxnnmh~rZD@*yP9ol1PsRi}QN(5d104;)*ZHsuZ z{4{APe+@Twe8y@`pYaa>da{3q@QVT#6Qr!FUa;^ZqePCEPmcNxMg&{p(J!ECK@toR z0}vVB?JM5iI`S!^Q6h*s{2ecEV2LE+ae8cZ8D|O^DDf~7HY}@ktDARs4=E1dTz({s zW5BZL=*WpvID;4Q_ICCIk%tA9S!Zupw6$Y5hmcVW4nA^9&}@Vu&SI}D?7Q%Hb1F12 zm21*}|9%Yu4{3|4kUAC=&x?W~fmp&rKT9A=rRtnKdYN=;s#!+*&~KbpNYD%K&>Civ z0}mkY?eV|+vUUt*XmHfRUl9Dul1e~8YhQr{0WN5kbKlMD%mYIqi78ShhKXfNWrlOy z@%e+eN?8B)$*I#Jo@Hw4{>iOPY~TJ>8@K19e;>3_zeaPip&>?lVhTkPjp%-(KYIJ= z%C$Fu&obaWz#@Eb1(}-UB7AMlj-jRTC+Ejup|7L=<$!Fyt4P2?qLZ=aWT<82)3O*u zg+b7dDXKZVb=cd~l7!F{x^QuhRq@3VOfWT4V-$)+(Wz;fc6)3rvDjH_4|{C7s!l^a zr=$M0Fw%xs5nUxf!^cMSyPM6P{u0p3R&l4iaSoJ!+Kn#Lo4QewXx@M-EE!Yu=8YN& zMwX)V$fi1SVQh+}Qv7nc8ff6x&y|jcvj_LzlCYxRPSULRmoZyhg?*(WpR4e;o+l46 zjk>(Rd>QRnq7FRiR8d4AJrqceA05Vru7+Kq&al9Mqd=R@gB6X7vwz~g3gj`pAI^f| zVVK}3mf}1&1a6n=KCN!PK8*8hzLhWpXsC1)VTutC1I!f>HvDKw_DJgwHA^-N4Adjc z!eWRG#CYgZ+G*4>A!Sw!`%VUifKZ7XA*K=`8ib9kTcb*!DjNbm7Bv0t{&j<=EKF?M zi5$H5O)XWnaKUO0Z8dNh5~j>c?Cjf};%9Jsrrx78bbcqLHP=Wd?^`MOgs8WD`{(^wCT>idbh}VkG#` zS-Md68P3xGtrE2i6l^pI2kRZbAJWIxUk@*Hu?0%hwr9G*hbV}K2t#Kw=v<}*0 zaH}IEF4Y@Eol_A_Q2Ug5u~FGo(#T~}RV!wHNJFDel%`zq%?cyJLc;@hAB$$0>G*3Y zOr-3%_wdeL@c+kI#Ey^9iDj2i)2#uP?BYPwj}QkxQ5#TiVx0wDi0(Te=ptCGLT>lmr0kRVCv;L^pYS6PQZ*c3Iz ze}AP*rH5CFR>X!wrw7S;O9!AT$H-$fmG&$Ba;E;pPG7*hVj9vhVqDzprC6l7GX`A^ zHJ?aJV??RJk}bKT?ap$sr@Re~u@{-!f&6+8cLj)o6?Cl5o!!OxUe2>~8lM93lb3}6 zBy0Ztme<>4dY)lZz;#F|0QTSTx-d2vI4pNt1st1ky8fJaA73rR5ZO_8uRju_IV0a!hUa%`c0qww)&lwa&*YPUGwia@sbrq@=;Oa)P=+g#z(aPi*o8)pkX(>!ue9E4G z(hpM0q#*=as&MS+(lxLJVY0C>iMplgRVlF2FqE7`>59>HgDup5>253#2Y1T-q7Ti2 z2E^Q9fwDJMmLkXD)A$j{l$o-x;MY*JnU?dNmuR$Nne~`eNthM=rVf&Zbb--t8;FDr zmVmukFNSIxBl3a&aW9w*TP$7g?~xos%;%)jDdir>rGGJHGM0eqCjksdFY=y8Y>{|0 z`4shB0zAS%WacT07OBCuxGjnsGFL$un`~FiU+QHW1~4vD{qFmxi)7|lkq#KmlrNc> zSa5K|w(ziLU1;w$Cx<%Sid971UcfZ8S0(=U;x;3nm%dJ{3gi!yj&iZ$#eiU2Y1rg^ zeV6-edm>-vL4z$R8*W&>aNNu(@!E;b7nVWS&lWp;a{c%Uj%H}diiv4NnHIRdQBx%L zoayZOGc_8&(|VPHfUp1z#woM=Zg(lH9=e^td}ew?a}vL7-|Gh_;=?Te#gqP~G) zxhl>Y2%e>Ju&TgT(V_UkqGEpJH?uHNdE%xrVJyH_J)W4t+KOSK;(@6wiyJ1QuQ4jA zs54ol3Vo;4k%6p@!O}p@hPcK~+2Rhd{$rJfeZarN3Zi{1=EqW|d5nSTz$;)6Gs2dq z;7g1W_cN|8#)N^vbx}b)M>QX_6C5c)Hhl4&$g;HTcg9x-L z?G~9Y386s>DWi|mtx;xp$Y+I)U?RkVW5g+gMS#1GcDw}@W)#O? zreec7fD_@kwW$&M@#f2M7zHf`+1S=UO5oIWd$pGT^3Y$$0QlOv2JFt{i3vI0+A4s@ zMRZH?#rGBqJo_)CH(45b^W3@i@gs<}_xAE9YamlyHScEZv~t9o(BFf+88^N;`TwLM z@4`eFU&RBkj}+i@J5(>^-pk+Wq~$ACzY z{7UJkT1&!`YRc3sT9|Dv+e`InW+fPZraBhYrH0LI_3>)>i$iNJ8B zc)ef=Mg&60rN_VMxPEu5^ogftOMkH`Qn%(b0LCkV<0dd+&TnWw;9{P)t>du>j2C2{ zr>ZepisX$HMLvw38mvl2-B5!qF{;5YrJP|-B1187$l8zr4k5qnqEnPL7Zr2Al!uB@ z50VrzHOLf2;NU_66o$T0?;{g_oqz|4phtQ>L~Bgw1B=qlDqa-3ADGRs)Qo49bKJLzPrf3uO+otaT>>TF=%u1wKoJhByKQSnH5AX36w z5X;C>rfU^{sMD8>mXZ0Kj63$*ph#y~pK_9Z0kakJ>HI5Co-`Cus;DMzvP9K;{#Hxq zeJ}Kvz^8-=Ui9o5670al*_Oh7iWrmFMcHIN8@eE(P!`69&Iw1Rc_6#Ky& zFaQ9x9hBVfO{8MSFEFDNLu_$F%is~qVQwqU;{MJ{G%M~t-N_{)hj4zu)H_9@STJ+)- z>>!u~AwZ3gXRav`^IoB*BKvuAzzB5kwsclr(<^!!ym$681Ld>l-Tox@koVTJ2?uNG zJ1EJhY`11T%9ifDj62rW8ZNa3ZWRlK(iC12X>l$1Uac(&+SJDjG(Ko$+1uDe>c^L&yC3E_9Ap0t z|NOU$(s2n08oXH6pEH9^BJ#a{X#j+-eT(8b1B$17YT%N9rgB~5vKVPJ^=+)A8t7+a zCo?;9@a5EgQo`rBE9Nouy>Ie%#k)UV%6{FtZ?mBLO29AsRPOJ1>@6kx=_(#K8Oad- zI-XZAtLs~{Lis? z?J#wrOqwkzSm|uaAaR^@k1CO-dX#vnqoGg1;7I54$$uw}RZX$wIexz$za`*%|04i) z0?Kyfqb*4ME;?~_l}@ksSws!c2DNvz6J@Y)b}p2xNs9}HLI|QICoLHzpcWR?(Qp!< zQwmXE-$m#%s50ja;?0uMl}=dR5~pa8tfD?Ah0#y=q>yq!lBBI9rEuXy8A)YKTu2o% zC<0Qi8zDv#u1=J<(gdZhPWNe$l+pq+X>%z1SLH-OveBhxfeDB>mi1psx+($b^r3ol z#t{*)j2UI3IxTfNP$(+Ps?v}w%cfD8-vc40E&%lfZC~#rPG>h(R zKZH9Vi6Kf9&$0j~FgJSC4x#Q!1hgaHIi%2!Xh#ACM(2%lrV>vi2#o`5v6KG}KQ>lP_g6w_FpwAA$vz#FzWdjIsaCF&?P6u5awGoA9) zkscboayo)6(UdJMavGcX!6PJEHH$sdr};m{Js!*-ialeg)T0@u5!frzl}v+_jfnIT za3$e{Vgh^W)bkdEtwf&cjWqPFqcGyqeECWU5o4{RM!)RoD=3BaPamTsh#RbAFzIQi z<-ogw5KUS;d24~;F@d*2y%5BZC`p34=|i`6XHHqVR6tMy3f|Wzb%L9xuC5%RZXZH7 z{us*AdX%r^_;wELEiHOLH}Y&dBA!K0N=j?y5H4hf=tf7fSj@R=JvZxrZ|$M{{Z~=j zzXiy*5VxnKKN7H_kjK3)=V4;h)Ku(LsCmr({we0@NCEO=M~P{HcqNlhvKC`HX0!qx zUfdi$y#kiPs91pPv1v9YEImySg#1|3RZJ1t8LO;pm>;WvidzLrZC#1#fvJ<)ifpa? zosO$CaDRUu8;v|zS$e1>g~`c0hxVUQrJXMh*ohT7eZBis-@7Mx{a{oHGh6#Ur4{*Yb_>vO(>N z8OW3pyZn%KN~I3st@(d31>)A9$1VqFD3F-;g5-YTCQ%m!Q@}x?2eR36%k?e=vQ{nMhN++kGpxt*CH#zD}`?Q}DiXxXJ0!4M9 z@Af+h)9$`q?R+}2r6DHT@w?KhKD)H6$9G<#eE{-`$N9NAIh+82ie!#YMLI%%VS?1eljG=Nn50vIqQ5)khuMz1rhTd?`xHb`SoXm6Ucgm#*| zn*3J*lL`ZvpdbTwPy&L-E{(S@GqTRMms$I>lEA>}a8?vb7@5kPA;z=;9I?-df3VlF z@gaYcu$8H?<6ENf|5EE}!?F)nNO^D~2N%fWsK$UWXT<4=op(#PD?oo@o>e^NtA0eK zupDgfy;C3E1*U?<;DbBt(6!P@9=;y4vCz>ourSU1(BO{z1k~pq=^KFHmtnyvS*&8> zk!Ar*j+3ouULn~aMkb5nh!VN!b#B8~m^a|=ES^s@($7fFrDosNMxq#CLmf8BDq|ah zrv&N|*uH|{-7hp&(n%$g+Yp%@D9ZG_H5XCqY5fXu2nrd@&bR%;{e!T6lk&B%6nF=Aow!o$x1yR&+_?rSoa zE~N6x;;|4_)en@aTAKVaeQxu)86q6kx?#Xt}TwqQ~iaP#dkhfUKg zcFQYJ*yF4yl`Q2I2P=#}fWX7f$RBcZ?Js;@f{nuDbE|cu;r!?e(dzvko3M__Lybb$^V`*N?YVYwfRZoer(sXT(Boi7Tc!mc#gM%`FK5 zJHeAx2lzMq>1)Zgh;3`1kR#qW2th$<+(7YaWcm5*%X#(u>gr^mj?yoNI@Uk8k4Ry_ zAj6sho|>H3P1|C<)zt(feV;C&7Tp%()i4?1*hh~E-z9x#XWwiY%3Tb2_93FQN#grU zJ)l96y}{@G^7Bz`$hWIUdJi9D_5aG4oTXyU!^UAQcjtv{0u|P<^I2NFr=h>Ycx5~C zhqK#9H4FK(Mb{}o``s8FK&Wh(yjjxwxnSM{D3_G_BX4L&+fdZQM7ob6Yo_z$`b~3Z5Jq2vo)szPWH=n6%O3Yf=n`4I(N&R$!aq0 zeR;8v!hZ!;Y2cg1wreKuFA~+RH~MTYCu}~LoIcyMtR7yMP9{a@wQ#;+~~Fp z_+gXpgO@i=T2ndCSQ4M!M6`eKn1udk7u7zi~Pj0@~WMaXy>V-6iO~^gbD&-jSM^V`~p84 z0T>#mM(`DWZ+R@Dm_abkmo9uMloM;AVZqX22X@Zsw}lA`P*|*MZ5j!up{c}(&1k{(kKpjlK@iC zQL~LPU-|f?Nsa|4omJK@FKi)V|IaUm{~2>0VIxaWsz!sg1?)Z7{U$4Db1y~#vs4!E zpw-MF4lWyN|GV}#d~U)x7h)+Wyi5{dTud*LQiZw|r;UtXWHXP8!0}@;r9Tq4B`NO!gpJBc*7!iKKCRMRY2Ca!Q47goSAC_CM z7A&b?6QD~WhUNJ^IDmMcp+q5l;PHB&TwD-v7&TP9Ls9x--r!;AvthuHs#>LTeMC!W zfghn#r2#JrA0k?}RHrinc2}==Su|di@?Cj%TZG!|ynS?lQlByaEHq~Llik<-m z?_q<`Ba89PQe~sTz|LLABcc)}T31Yu3|G%Y6vrX^fy822gj^a69`r+wSU1kE1oSVppWl=!3W@vh?qXPnjW7&y<7V{%|1Wep+-^>536Di1};)2{g5q% z$Usf-6PEDeIQVn9J-RRf^**eOP6SwW7;?sOU`JazL4p6uUjsDIU5SMm2mk5q8O#a= zfPPOnAtIpvDOZ5y44A2c5BhCGJv4H7{V{|hPO@gfMg=)W95K9q z+n8K_Hbt|T1~Ckb2~3+c6dp33xrQU2ULa1gKr$IbA;01kz+T1+(Z7e#MEn-m(zQf`UwTkb<&r;^a)DXm@`2_4H6 zwKOFd@xzN~R+&3LvkYY{42)!ut}gSE!RM>?a)-|QKF@5uYVUT*5D0JD^&{*yTk#^D zJ3m)Up?m-(JS;_;1u(IX4!h8L%y{Oi7!Q+<(8SNMcYJ@hyZsWwSiGVHGZ+MN1(3U{ zoDxaqyx3?FlrQ`045e5gZS>EWl7%Iibcfd;q+{kz79e%RqMTz0uqLpWgmKU!8IdTn z`X!=0Ud3`XMx2H^Op!nL;bMJ2Q$nKx2P6XeH9CzcPKgRqrK5w7+Wq-|6rE*QR9zH> zFO9Sag0za#p>&4=A}!q@(%szxN=hr;T~b32f`E#242^(*RcjlgR z_TFo~>s3m=ZS+$`A~O7!vfn2_%A=RhAwS=({T3flfynkCFn7Z>^XE<>DtmGK{Er1oqf zSoCYS+Yy_ZmD<_Hf(^5CaJ5>D8$TcnYi~;`A;-pGXjd)83Be&un?hFDI@uQ5iqd^N z%IA)^2+t=Q=+kNZ$~kb0|5rHPvUSD*8-n;%yuD^c#T;)?XWNLi?VTEt581|7TyVq;f4z>uCEne1YtQC}$ya-|EdZf+sswu^$u`oZg)?%~y8h3Ek+PhF}`op4%c@jQ=w0;+gSXwa1l9EpZhC-`N^xVTEpcs*9@z5${RS9OUQ zYtftXT@qa-Lx1g&XaI8+*;S2ciYY07Nd?p*=P0oJsc5XcngRR43v=|eK<$Vc8Ma*F z5Lbybld!|9TktQa?4@yMzR1seCM`?@f-+&Vd;BWL)~4=s3W-IFE4%+1Ev+9=5?H>l zh%u-Zb)7Mbli%}b#ehKk@6{Y_21po)B7Q(kgx&7o)nu@KjtBG@?hOs9G?K4mCV;hV z%qAtbx|-g2&&od6BsI`q*MO5=wbZas!%(bk*J}VYf&Fz)53U08)%8}>o+>^kBf`1< z#z1N5f5xB(^H`eB_J5Sf`UqdMefI4h)2px&+tP`t@i`M6gMgkx#X26BJVmZ{Lb=Yfy$s zhq!7s{M;ct`KM8!!M5mB8&<+OJv^-HpSvW1cUD_3wor0L68-2dX4*PnQ*r4S=9*Ay zODKu8(**^qmurC~JhERWLRMq8;r;()HuUZZenIb~hQXVecHP!Dil~5~?oq4l5At8~UtDdhTemMTFa#X44p{A% z?Sl9uH5^4UK+-3lsw)d{gzxV|WI=ZCcLF>N0k?|x{z4iPm`dsnDs=6AkUOMr^Ys6&uzDA*f zn!3KeCb#Hy?sRxzZ@IcU-9S_G9L4Ll+wOW7*74bu*Ch2Vxg0c>7rCOqF3dz1ThEDy z-I%$HpC1}N=!x4jPnpLyZhoFQG|Z6d+liu>+mp;(RGMI}C4e+3woW4r40dU<>l-=6 zC}QOw#m>{8_=P{Q?C-*g5h3x`TX_9sz3#HTEg-ei&0lLB7c)_v_4P#2>@LYg@68p} z#`@0hy?yW_^0Lmw7wRu>LnCOid)7!~9?23M1Dz0qXCB3)VYm#)|e#& zBIdC-l0eFDiU+|Kj=zU(!4sLknLC{?8K{-2RIdh4zMY;4dt$v+N|MwcN!5{AO z!iHYCygq-v8Ka^J2@TGwZ~s+O%@0kdHJ>Fc`PrUd_3?uoouH_R)b#WQWcNn08g&84 zl{1dg9lD)h#BY(vxm%ju;yT#uC~yp`$`RX&ddB>Sl)>r8;NW7D`C&pmQ<}IPQbx!U>_uH01&6Wv>YV_zmyIA|i*c$q*=N(~bG-_qzLKZcut+ zZEeVyVQznY+r!B920`#hw(Gku#B|arsOT5IqKfWCP9m#jJA)4M_bSm@2gMWXPLB3S z?J0}Xj46~5&vISBM!^I4hLtsal|7T9TVID%bL|(n*Vaj!@f6n8xve7~>(A$Acab`4I_a@2l1$hcD`gJzDE?;qm zcio0`Q7fD|m<*fm9$qS2?fh^bDO=XYf^>~N+4?UATp6`eQaLjYcL48oVyvz+$Xi9B zbT4*7ZQEhSq|@eCC>KZ9gJX~i2Bc}_TE8;nCYq;yo5CFA_s+Dz8rqy$eZ7$cP*dr- zfsL*sa~)Y(;r{0!9-J3ui@fn(($YozJM*ztLz@rhN-@kCiXAQR&67Hrw7~f{hf`a1~4qVW);9kt=VfZlSiT{=<+x5ptYmEg!kaKFQN6d?8HeIR@Bw`UX#UQ!RertQ*{o^ zVo$bHb;qr>udrvjR!w_tsfnh^?%y8_>L}geo5C{Q;Qk`#rNF?GoH(U95{FTnNv~4- z!zxqnYD2vSn~El&GJPx+fKptn(VmKV z>Rg_~R~PIYlggO(nfc>J^OeEvxx3Dn;}E)=55=wKer z6~hvM(qN*V1JOf^-bO;fa|y_P^e13*XWhwn!=SdS4<7}s|f5K5xHmy~)F&ee~B$>n^HyCW2U~m03 zJU(*azWdiya#2g7)oQlrwDM{2abG29kb2B?c6HB;m`JwxquWy90*nTNtBv{o&HEk4 z&6PCvR9qRTy&T5|6aTHyxw%LN%D!W(*i62_-C?c?s-c^W(WTbgw(A9B!}Ivt^&1{Z zT4Jc7B>qmD5J*2HVw|q^I{zKv4qvEu8b3FM+riok$84C%JzG2YtmFI-FWWKl{MJeD zPmmXCASNBqXFF7aHMzXbXQe1Fm7wdfogpj#6!S|_QDc)%nFhagp-dq(fABDiz|?ZU z+0V5x)``h?V87!ux!CH#$z>1~zAR-lg+kgyC(1NRC#TUtzkc3;{KVf@{&4g<9*HuY zVAZ*bP&Ul@_WdQH)MviU#i`1EB09aV{%RKBH#AE-(WgRcRr+F9(^)W`YV^tBK&^*J zXPnA31{R0ibWpR$EM8@$6nbTVB{Q>%s8>1XX!TmoStaxGXj_2sbl-5?sX22994lM(66S;Li?F0@tnNb)v9e{y@tc=V+E4AroziP zT2<5b%WTD9$u5u6%GwU!y&C!KQWdwMfg546H$_|LYFS>l;LDIP z+9xPK2w8KcsfKKI=$(*42cae}&*7(Il%dj-liO|kCFX?o43|#hLk;PY2N(a`=TvI+ z8U^nwsK`g&J4E=n2tAFwXIo&#{oSJZt2$@=NG$nRA0MIAFW-&cT0L*(xHI__jl+<2 zXSG>ToszWTby&V@S4c01GOilN`(l7FH7W=Y7rNiMzFP5obg$)Unx%lj-mG;FCItLf zKWb?zra)W7QznSJEJfKZ*|@Htqo|@@Hb3thGV`SO^B}iq@Y(f~|J58lmX-W=l*ot* zXNRD@MN)>=+xS+(^r0-6FpC)B!=g9rbgGC70b1yfy75|Vu(D~RgQw#1ijtMeklPv{_R^rST!H*XzTJD2zYDnYEy9kxAGrvA@NEXkokmG&H6QJGAoE zL*yHLHkh+jy~n0tQa9sbRK_mDna-Vholf^4Bj4GGZjT(VqdW59XwU!$aw3|h?fv;$ z?{<)LcR%%wj3!`hR8BjX=?^pFD`$wie@mhS)S{lmiX2K~L6qaURFl=s#!hY5ZdQY; zWp?w{JvgB6B8Se!UFUDqh#ySe&Obw5Vaas4PTEZ^VMUQJI6)!69n>Fin%q?A{yxfb#0;1tFSiQ|YMcA*tY- z|6k|P0bi|&=O7lP<&%5x+brg{iL)-kZxAzC$=ROMACe!AyP{{Uq3*IoYZ=~|hJxMe z1_i)PhUIm<+uBv6>nD{wJ)&BUm0D{|4!X7Fgv4e~64PcH{ltOozux_@ECf;@?b z^0?+(of1_meNS2uS0~$-i=WR+M)ZBIepCW;W&Pp`X7My5J{fjmW@aF8DxvugM}h(? z7+QAHK2)Pq#{m5iG2~9g4d2QcfFbA3R}6WU{5Ms;#Ww?s6evVbaWG-awhB)cm6Z8HACi`CfiWJ7a+5c}_TN!RJ!{ zabza#&XVuVVbp%R`C5r_ZO1dGx!=Y2?GYkWIjFrBUEul(n*PyVqHK7*>V9q#F!seM z-Fap-XfrGDXr}ho=PY96QR2Vvxy$rH+n=;6g<#DFklEO(fCc>OMWMz}&Y7=q;7-{f z!{zGWGHs@^UnedZ6iV^hZ$2I?+gKI)ECIWMEmH~dXDs>Mgc>@v*`Rwo&m}IF&^K|& zj?+I}LJ6zsRTxmeu-M?}($eKm>)-{C~h6(5Fz#*_`|1#Ik`k_;y7DNSGO`nyN_SoOMG=ai_tySOB#4>_rq$RMBNNY5yzmhIzZmKuOidrf(>)iRGq$`m}Zgd*XRR7q_O|llNZxflENEneH>}~y7vEeIfv5X;&>$ZZceGJ#t4pDb-CrR z{f;gP@rc+NZZ0Nu6-G?pC6aWpXZxlXrx%6}*7J3Mf^gD1K~{8p8q^eFJ0 zBKBV7weA0#eZ~#fsZ4RtH-E@SQuuzjnYpp&(9->_HEr5P9}~+{b}B+B3o0&ld4utG zLHy^fFdLq$E8H0KeV^r9`^lmYT>I~nTXtM}CL-S_{nYlp6J%9A@2WzLrI;jIoo4BqdU-_cAh2pZB1tI#d<;%(!V0F zB1)&wT)(-j&I}m%=9`g6dUlAKqii;p_qJ}s1%P7swOJO zSkim?=!t%Ul+R|k_wv!%JR__rv$kt>RYAsmV~Yh#+QdW^G^W&rtMayxPp|hq+W_$k zNF5gU1CCY{YFhqc7R$OK9AOx&cv*gS=0QtgPdI8j94Md}x?wQMQz?FPN+M&h1I2)2)^y>wxCh_{l7G4S>;4%efo;%C2#5iHW$`{*fBXIrp03Fv{O)}Xr8w#xW z_O6HIAHCpGSXfN&8)*cf&z#1YI=?S&N&zD)2&6^=@fXEIU|%;Wy&J_P5!bnowQmSiOC4mupt9})L5)fkBmHWnp-YFbaf zEz+jl5CE^!H(&;i#D7E{hbhhc3sQU=-}}F7pu7Is;7 zCGXAZJg%B;?Q)%R4}&29K?kug&-$kt?MKikWZMI2_njM3sN0Ycz_`~-G5)N5UB=4u zX9x^Bu2=_NQ}NS(oz^))M_Ca!-0lCaO@ngJdS_}eS{S)SZ}a_~_Oz!8zqSe}B&KF& zO4u(YsV(_lt|o2<9Jj7aCP!j)hjeuc__p>&5o3ltTaJZqHq&DK$uMNpd*=KcN(37t zq8f2ZBbZ`3bO{2U;pcNU&`tM*DqaYHp*q#*n~m0OUC``h+Sm`?pV=Hh zJ5rL?jJNN1Q86g*={39GxuyfHU#6QXl6^;NS}(gE4?!-!eA5?#H&qDJs{H=tpr9Tg zAaK@~*lerh4m>?2B?RO@hozv2CIsfo*D-|P)aA04o#0rWQ#s6d@!R73tY*7hot<|B zdX+ikIxwV5MY$n?0`!G|6B5=Ke4OeDTT{P%bExHShCkP7d+D`cVY&GPmiO}Y#e_3x z(?TsrB~esE*Tm8=ixC6J=j4mfUbPA`sH*i(2cLYFS5NddF?GnAUhGO&yE7D`p}WxO zzUHTHDZ_ek(Lj2-qj}liZrX+Fr-iZ*J1DQ+%tHBiy|w^)NI9qZpgGeC?~h(2%CSd* zJ+((q>*@!V5D2Tyc11gOyA{cPd;a@+b{V6)P_45)cmUu_ypQA3bApH2Cak%zLclp< zKODA#duhGT#>RH>H@tM&4;kf1o1X0gG9 zM^`>u@XEi&xmoeIh>khR@!2ieG@+jN@VWi8!U8|(F(!^Hv9AdZCH6(sF1x$jHqtt` z9E`9H>PJ3id&I+2Sv(bhzC16R>{sWVTX?(JamZ4rA>`w-{$T=7CHL!gVw4p0$7{dJ zuycEmL4v^BX}QyCZ=a)vT0`&Cnj6Zw%lB`M@o7_2b*A8+YwLoXoTX0XUPa{yO^_^O z)9bq*>!E`U+I!2s=PXb`h3;Wtg2~Ui-6!c;U=Pgk--vo6GFGsU`D>RL7@cwmG% zzDGbrW8}B;BwVPv7QSL?x^>EKAalB{DQw^0oFJ(;TB@t@;sv7n;S<<`Ynx2%YA>f( zX7EzgscBGGK_mZWu@E;-2+Y*D&hIjR?q<4uUn=nXmsDDW;Gy8|qGirA3-6KoEc0Ha za2Z^}>Eb7kV)R-TIKIW!KD(z_!fP@Afb4dP-V`^=!e7RhIWoi)8AEo!b_mY9tT{sZ zxR}@hS#3T)_k<<7{;=AP+mH%9m z<%ND#d5R6+d1xjwjU-j4D9$VYkr+RZkBuq8@e5S`d$@V^{-oQcvz6pF8OwTpx5h#| zCe7O`{>~#vjT}qCzwC5+Ri#o}XCOOpe9HIU zSksD|m7ASfbafXTn=d^UNWqu{D2v>tKWIW`mlaL=zO9E!{jUOTd5UFa65n~rNd+oC zfE!Ug#GZ4Lb8-&S??miWZtG(}2@Amr>DAQ}(;k<7j|~mxU3ZW1h83kydi-QiXt|`}_KrWzB?BO9ew_9S(&_1$vNY4B$;GcAwC0AN}m`U%? zs@I+Y#FFGey2WI@_0azYC}^h3Ds<4%xYLCIY_h5!oar55)P$%q*0#C=43X!9K5v>b>Qc?{*{mkYq{gv(-toE9-4?Nmx}^uO);@DhHQ zH4pj2%*AI%S$wJzwlkEviyg&&LF;!mtE-j{h7yvU+E-rFHE|m2YmOJ@w)ciX)RA1Q z?4$RI0gM4BM4CxU?~{VYc``~U2qLNJd^=UyF~P0?#NGlxBG^ zh#=k)by9&HvL|+4evCSKFj`tvG@6GXrWrkV2~3{Y+LrLZR)06tHbjuY&d&YUTOQ1O zL`&<7oP*;Ga3Fs{E|=EKP321xC;$ybCp^j4k$7ij3Y(uFx7rtRv4NvI?__s%R8d1- zd(~5ab*)lKW^@fnA<(~Gy_v<9QNMC?G;xlT$AP|?wbU_DK@|umtnTorQ{-ahBNm}~ zn^BDJdK{^?#rBGH6$mQjX#QzjJ5g5eM{&NpJh?yuSr$h?fD6$IscUFtnTmdQ*z&tP zt;HkVp0YJ)zU_{@Yn?r2zK0FPilnXc>-PbX0aRAOIYu3=q(lIj-9$=I$Pr0dwoGi< zIthYo#d(`axd*ts*jNv9c`mj#d2q4p3lf`@?gx@UUQwk*jS$rAEcNd9@VkeGif;x%g!vLoYB}=Jmt3%QTj#sm@~~*a6U^W+)oguch8iC00-oJzIkKR zgM89M?RUzJ5pshvM4fJpm8#jQHrLd#5<=Y60v-95VXepK?MV9dQ(Q`dHdFYU_}*y} z#2uE?3U(f?atbDwXCFIWhrxCo_PPQcmjhb={%$}qI$gSLo{ivu&zt$Tgww5xwSU51Uw5SJ!XArjYM)5pP)2 zAeK$NCO4>qr5$1+b=Lw;xpd>R0D^>2r#1Cna|UyBTMPAlgO$0#N8dnW<4vJX!mXnL z81$$hZV55wy4;cIxn2qX6qfCH=v5Mt+#_xn%i+C!)K0x7ViwL)9hV9yaz~FXmVAcO=PH-8{W$G$3AfWma^dD_K1h zSmF}37%otQUmyNmc>!D~zyoKUp3Q`x#EkI}Xvx~LoD!%Y2n+7Dv`H$>zrEXdd5zw! z;)MR3xBGTVhSZJaCA=&UTD7q@$X~QK7)AOa;!*9NW&rF#j7Nj)=sh$ z+bKK9j5_kR7?_PiW9fy;CTueem0Me9O#?5tHaerYtd!zqZZBS_sa;rHHT)k}5~n+K zy@yI3Y3nQ+sMJSO5St@E zf36Boubs*YzvcaRS+d4wD_|F7XxTjW$EB?nzV@*`NjAF3Kp99U3Kit;Kuj;+KQ?dg z)fcbh>7QBgc;F$*O?HGA2Iaje$7FE+q;p`*NYM9*LyjCfRG5fa4yX|lgG?dFtv-&7 z00%Q4mPk;0x;)BknBVq)VuN=qNSV-w^7OL4(AN@FwH`#GT zQ4&N9?Qa+wI{(l&?&PD3H}PI^%LX0azAu67HyozEoqwGCbQ0>W zdSGa!GXLD>)t5m^-X>Auvj6XG60RH3BvXUi6~fyD)9vqP#}PO--2 z>+kM`xWrsTJp2wzqdqGSa*czJf1XRY4QDbM%{sXmiUBs~CEg(YnKT%yR8CmKE;%aQ zr2PE1>ie2tOP;?{5yTJ9*F4Xc(WT3b8kLkG7z17(2_)=39?K;gDm-x#{D35W6;04# za>qJdI?4NUpSZc~(_79Ny9^JzmG`R`s()ulw3ZOKclKOK4P=!#NrRIVdJd^-u}P1} zWL*m{$X#Vy^)kDisoYs*WlhJkKO_&VSvIrLZN4XaV=n-RNW@UEGPU=R<9Q-804XSB z2YX$HbY1hC>U|;&xu8!1VRs?@IO^zxYWFWQ|P| zBtX{_5nQYp=d&0lE+uK8IC>ySWkP0wlZ3v{2|bA8@$FU<%@=cxO%f zlM`#el`bj{*Vx2F5PA7WINstj--v`}<;3klA4=EAlMl)QrtBcg8Co8R;$cDgM~U_I zKF{h#E`a5(BUI zg8+fc$0B~(I?_MO zG&n4Qs_oeAxl5I^V^iL_GrbcXLjX2XYz%*4It{kebW{Fu>pgK~y+(SxsB`be9v7Em zX?zFu-M_#S-h$rnhw|!}oGqcmtZ}X&+!FqySY|WNOkd-s4YkugnO-2>BFC}%$I4gmEH=jjF^@-HW~7>Q{KjW zD0mgj)aWxy15w{h-sGEo))|#RX*(`-5JK`&z6r90--wPElD)CID>77Q;zUg>=yDsR zE2kQTzK~1`8r{?;-nQ-6$EzC3$i&N6>T7r6Km-JCRw`cv4>!wCPpdyR7;`gt%FF2A z4ITYgx{nMnDJ~11mZNkCrScL zb)7Mxa2fYM(tMEk<=;~l5+p6rd+yG^{1ATBBfF;h-fe2BpcP`Ftl;IqbhXcQb*;0L zmAW5a7hkL;Q04}sx00S10%li%BIgI6!^+x=hoxHThJuXPAs|>zkVS#MmGx41)46$YaXx|azD|KWDdQ&{KPaIWHn((i|*?FGSw6;v&7qU=h zyQu8^s#@#d@bAP=_FUX$_>zjG&eOM{Uf_x4><#XH#J-ReYL$_%#34>zADl|gPWUQM z{MPM}s&HhpDU|1Y+vAvN2<6r5?sw?>KUJC0YG3YgZK?b2GN+$9F9b93yg668Kb@BH zz`b4|5kK>wwwUwv3np-1i{Nv$eWJ(J<~uXwEDb4|}I9uT(f zE{x_->Y#k_&tQ#`khEJVg&JP2IVt(^EjH!d+~o97rvb6l`&8BX@b!HKdHcz2efX`J zNJp{d?G+B_Qtpfsrq@(qgqXrn4wKq2(3QiKP8<6J6O=ezowsZ9yWLn{sp<6>;;v@p z0Af~5D5HR<>+cm7-cCQ*#CDJ!DKl|pQ+7c8WIyfP^={HPws7S}-ojGBNNvNx8qna)HmuZ7GX@Bl>5NlNt6}(dy+)8yb+zrLvsC<@ z$R3TzeAZ}y59_@~BC{MGzyGZk8`RKI^ubfW)5D-8#CE-7nb>a;%x;&v?2$?%;yo`x zw5YU;~|QOYb0j+}^yYX@(lSkXkAKO*D&%&%w|1OoO@m zDw)7EkTop0v%>Yim=35_A+j(K)ax941cBj^wAVdZT zfUZ>2R@=4R`sZMM;}}z_c7O^GmHzDxyeOnX{Mxn4M37 zgeWFkbL*SjT!ee$gww~T8RJ~~CPx4YPK0-7IXtQ`8C z2P1~Ha5w)`qo|36u`$oBGWXyd;o^o?0U@!N0@frZ;rO*hS-U10CeVgQy3@hTFFE1M zr-QIu0a%9Q#jbtc30fM0tgg|4{0{n{5j6k^^4B$pwd{*UYo}xM#{G$HOZP0N5gxdj z|H@XBY0&;N;MzLJTU{dr{3xAro306c9T`p62H-boYHX6a>@j37t_Qb~bl%$fCWCRC z+fb2|RZ1#l;l-2t-LCAZ6SK%LRF(=n4P9PtHD4!!e9%n^P}ud6PM9?QCzHy++hk`T_MIXOa&N6>a1PuF z&3G7&0*J~0N-)@hUml(VbZghq^$T>(VjM~*0KF13xXE+vv3sGZ#z6CL?tvE)ep28b zbZzN3G@emE4hoOM7I7p>myI$cytYH+KD04(F*otm7g6#+*Vq-!;T*f@orgv+E+6~t z?8Z9|MWq?Jpv#F>_8^yUm!)4abuZ{M(VmyB!Pl)WV@aN^Pbg`H_Ao*|t}XBJSJMa_ zHG=OBMTIivRCpwVM@qQ-y>~@FTTUJeR!;pAu=j?#JM=B89F?M#ju?C{upqtGU0QhX zM9Z@gL4nMHKmh?ULp3sd44cM^;=yL9dwB02{cuSwJOCHsx3dJ@z$;p*9nTEM`Tlle zrCh%wTy%K1`H!DtBRLI`JWgl3o_&2d07ALHpdWmDlP>;C7>;zu)<8#Rhk_ENtG`C77qmh8hZUPT{VhI6Ft;f8o-N;y0a_8z&}1VIY> z9Be~5BF-Lx^Fs9WnzikRjoKXNT*e*#zf1q0{5sYLXd@(Rntgv;uQ_?o1p)*Y2ET!2 z)YQ$&ygP>3RR@bVBihV%h|nPWs1#WN{=!uRZr>aj)T2o0l{vDGFMfXX#J(B6;6wv+ znbHJZnZ>CtC(p%tla+&hz=$b}xaBg{jNKC+VbmO(!l& z$uBD71_IVhzlY6O2ptfbYcOoRxYo~5VgB^)?K4fIwyX?_lc$Du;0I?%zepE~ka(P^6UgFieatEgt0J*p&`v!3+G1HDhcC za&EZO{lVIm79U@-(G0L_ckOQG;tX|bXS?gFemdNoCT!6PArJ5{{m&S|A}pBTJ$zmK zftgNK*Z2jWViI;J1wTEBLRwYXp7{j4UqKl@Z~4w)_Nw4mh~A5Uk4_3gJ8rz$mOJ`>aQ`RYF4g5en|&wK5A zwo1%4mN-OJ+&j)-fIJn#2og5W!e=s%=Jza%_H!upH}VwPFLP={{kyi zomG4zGgs)3Bj@N`OSjeLpPrfuJ?5lr^&O``|x>4BO+ z@SJMd?Z9$o6K!WJ0PuOaF=^G)Hj@FsUZ&B$urF0IL>wVY)8uzNolPon`s--hlQLC; zOg3()8qk$9=eYP&J2s=46^XzrXsfphA6gmRY6NN6t<`}y{f^&WeJ?|xu)ax^jx>m6 zy-qC{w8|9^+;5zX!T0n6Is@>ofHlPhyqdM?XQnsro*Qc0E!AR)i`qIkIodg8!hPgY zMnxB$l4nIN`#YMwl!Ba8c`2^kH^fUPfE~^YSsA$92ebZdoMrmv0YOxzfikU-GpXDo zfC65d0Uy^4&rC;2r{gnYfS4XHo7{mpn8IBv$I86hByxjKC!)aqH#odqFQPLIAXES- zLlKhqwYAA_@2T#GSyMwHi{ZC%@9)Mr&$)IAlltwp0CaxH zz}?GE6zP2`%Kci2SdsXGvec;^r_OUl@R{OS4mQzUzFzt4>{cIpJrK%-|NaAbS3qs` z<@bFPU-wGey(=!0T+?!`cMpicwWzJo2aZH>lV!Dgr5)F2rLT*}^APkk&J`x@Kj+4Z z$Ls)94p1!U#m+HgDdYqMVE`aDGiIqdN`P*(Hjo)HZY(%63lyfSv3EdkA)yH9H}442Xn<#420+_O)7 zXj06M+nK94un;Y-5GN`zCBm;|3S4`GBX zzseUbnRJ9@LGy>)y_-&W7b8)}ClCN6tYvBA@FHVGhsI<*&eKjzEQe6II z2}U2HqD!9I>!xR})y+E0x9zzI(@oh;F{u4HFrQd?u?N*TY&EBC47_xLzL34ZIPXz@ZpQxoqIKEM@( zYQ6T`*$^$AsGVj-Oigy7kN$T08>l#*HS73CO%ce6N3p= zYIz9vILuD+x?an4+J0IoSt#o@ABC`#aTm9bN~L7)qBHEev-@Kfz1qs_+yIGtQPY zkio?qvxOrsoE88mJpYZx&k%(LVy>3EO8?%;_rN9zOOi&8ez(lM%XrV0_A-pTSXqYc zl|$V^gl^B9wpQ<*SIYUaDavvL`hVcBv4w^E zE+WSoQwgoVjY1^pUfj%?5AH z_EfTI!@7sCnBjbk|LKq6=U3Ml$v=NbJeDTw{@N-R0bsQF03|myJCmeoVxXl(Mih~` zo%t%dqG~%)I4z1L`Vm&>r=OD~D|PIHdT)#t^bmPtJHI$yV_4K8tetFr*0TN_86m`d zQR1aE$%vfx2XS@80#~Zf|I56lP(W zpBuAv7ASYCudMJB&UR9A=P1`W|EDpGNMXKXuaG{l0VQFlO>J6IQBoXuQ#Qg3U6={z zN{4=Ob8*>~*)zYYudm00uwUdixAj+rT_=qfxCoCseT`f&$zwI6c5MDy)bfD|KYeh~ z{43^Q1+iaJQGkgl0JZX~G&MKlEt)WW;%147ds#@NsXdk4+fqG6;njBJpGLK@X$aUX z(3s_-@`7tM)*w8h8`4@_ccj3E*U-Q2(efaf@;o~#tuzfRg zheiN5(ASp@@)0*t(o_oaRdjMvWhO6vHNHctH^dF_A2FBCfCCR(pLm} z^6AOiMp~?s${hbl?9&Gw*uZrMci-W~+%~O%et-SZn>0(sumc>-4KKnvI+v_BwOm|m zzzxowK0iK_VXBdloH_K>A;la#c@u7+TtdA~imkmpDe788E<&1DFzIsg?n zf6ZqjkJmb_VOD%j6DK$&Wa}{(&0tA4s4vzoE{I3Q45b_DeX<;d8H!axbW>662O%$oHANvJ+CyKpk`AN9EF4rg93a3JF@j|NOSIV?ED?qa^Dq0L~8 z-qzO-g>b_2o3Bc$!a538!vK1EAYt|`aZ9H(S%|u#?bk6IVCB2IdYbzBXTsOvVeaa8KXl1w23ln^L z^;>5F?2SUvK-2+*nKE?x*-f%XnzKbWiWy~3b2Gi_{#>vXee|~YO`=|nf=0^x8egUu zgtgIf9oDg0aBapqK5ca;q8IFb8)-0-Z&*Byu_9*D-vEd8$C=Pd??+aJ3qO!b&8WhAeRvR zs9MKS(0g{5A)5JFR7~_yeZwhHF9swl&4h{dc}TF9MS4QZt9fbbXnn(qa>F|6u|~t} z#xag!^sgbo7Nf3RPGcE!lHNM%7w*ZF7B%%1AY>+qDVy6gsxSW8bE_@AG>Zo{YRES* zq0vbjHm;$;K|ajLVQz1Qs0Ll_k2;kk*s@j)kIBt>ae-OYAY;JPbNv{bEjsqOFo%+% zQHsi^_w)312F7%Q4^C*3m;<}iM{;lgH6q5GHJS+b(S{expc}uOrL}#wYOqtzx6j3n zj{DIsso#TRLsF7Mwkl;@y-uarA5u+eRTamImKnG7Q;}}iyT+19%)%sYZl0~aqA&BznC8q6>*=0C7?G>>oT(Xt zO)Ui)BxD8n1!XgNNgioH2bZj{7-BYIB|eW zbvr+P^(0qn6sEflWU03zfi?3xF^nGq-@R@qP*1xPhTOOKr1m(GJ0_1J(Xq$&Ma9cM zREk9ee5KD5KPCD+GZf{a6L~v_%ndy2tF49Kcp^=D>JnZ(*RTI4F;w}T<)NS=(5l+f<>Z#|9(Dv^zO4Q&BNKZtI?%rALuS# z-~-=DZV8dcpn7CHV-IzKa-*6hX|^g1U(!_E+Y4$ZgR7I2AJxTRxgh(eLvkr1fjL+;NjnmIVBCD$<%H1&TY5Ht$j} zHN4y`IT=gLcYGZlWAJoJ;!CL0Ex%=+VdQfUqq~Yp+1btN!bS#fPR;|xC-yfY_AlIQ z6-JYj0w|7dADpCpN;&U8yaQ~)JsL5^96|%2@}iEUOiwKlzKj2fA_Y^Jtf+|KP@VJH z6K$hqi^JTHsWGOv>=a7jvPE))aTFIpOdq+J-)>En2t1`!X7ZqTdZ%3(dW-$(m0mOv zJ9%b`mOsVJD_-h&GjUAj)63#VeUs#UKDu2QiY4Ojs3765pYuMxjAI*33w3(%OC7L3 z)kcHyJB#nBKBf1(ABwh+cm6rx6zJSjkfq>9WFLBcey3bQOYwO(gMYYaawC=c z6T@ne4{^f3-}f4E$-{Ce)+{?M?;O8djH+jlxua?JqF%KA$uAPDM7Kj58R*lj`TwXn za9@z!4DBo>DYB6-Qeu4Ey7%)L=$$5PEjlRl?hc2t^u54T?Rj^X>4Q{pPs+=R3(7kU zrGKasBi!O)?`++EJ=AXPm<>Bi_r&}4_v+lp8}Qt8s>yC%&Ao>5-;ya*u-q~IixDl| zioJu6js2J-E?8Cl@tA4;hVZ@OGEA8#v=kK)kH03e zEhdT}{wK)+tnI_s_u~4>2VV7E*4r+=_i&~8=%N3e#ahvQ@NZJiB&8IGwn@Ixe#*Uu zRfW{IVE&KmAq>X^o2!u~hKs_H$9 z-r>Eu_DN~ORgH$`m5#*(6OMUZ$!MiLX`Nc^WzJ0( zY{`VDt?up(hir2*Q)D#uvD}F?=@|r<>N|Q@TC&>_V)#+Ge?985< zf8($7{SA5fQ&8my4RLGAcwjRX*8B&>v;4BD`~6+cq}$Mu5=L!qf)^XB;m};QvoD%Q zg0?w=9N|Q&E}YrnEfY+F8qPP`G|~Id3t@A&gUAQ+JQY9~JUx|@Ddi9f>Cb;<(-nQ9BOx#o#EcUB)#5mVxe;i*Vc? zi$ZivPpgW@jS%IwzhjjS?VzkWEI*;a_=sEEjt$>8!Kq-&!)I*aTHpuap^_x`<_G|o zyu?|^3cCTW*pdqEZxjx=I@L*W;1FAHD5bGr30;1-_aXC$hwhKJHdb&)U@-iNLAw~u ztRoZ|1kwHO+-bSj+s0%WE0ISlzZB2o{OkG1b8pC{AaaOBHpa-iolv8sm(?5152LiX z+lTTVh^I^eIVpBv%fYp?wl5OPlykBi3mE;E53xf# z4)4jXB#fJp&;>GzW*rcsm!@iT7tc+ji@7GX;nK@G8-DDEr(PiVqneP>jf~fYy2p(^ zftu(?3Pomc5jd>FjTFkYgrgHg>|7klBhUy?bJOiIKge8N6;a3|Fg$mN)r^TN8eXOGGPQtX!8@?!hG zbw4NyHrA2@&5>VvRIs=sZ8@;#I6)WO9Uuc?!=n^jYWf8#7^_9G+I8AE=0yxz>S(ISB z(*Tp3sDJ!~by<+8j-egop$%JahLFNd#kV)w?5)WXde6l&}4pXd#tyhxs7EZ5o8Ay(((53Q}G9f4Gj z#Eq+a#-$W%1RTB&xxmvSYI@ThxgTxeGxxS{u0BbR(zK84g+kEm4dy+Nl~I~)xJ^#x zq=G`ui}>{J%y6JvIw|mX0ytH+f*-S?CA66#ZxhA^-T+U>+E#h4#7{c!n%|MIsA%DP ze#OU<-IsXy2by!7V$2SGE^B(3N%1zmMbdD>9T6d{26G@lbj45ODh$AuPUHGv8ZPL5 z$8mk%n$yb&)RrZbAHiNOlo*WK0eumy*#w>B#nBl?7LfoeJV?skfod7{AwQKG{D|Ra zpPzbS3*6=B(3WzQG@!c#GtIShc)Wne^4*+}wzDySE0{O{YYk z@@4$o7tG+LM|ufUyCw4#we~s(0_k)hM0mZxjfAqd{l)#ZES<)bB6r_03O!YN`JB9y zv6XvRktm_2qqkNbdbxDn5!E8q3 zjpZkZGLWqxl^+fqKFDP^cS=mNvP~8+M0eM^ue(~FG^exl#p1&ZMV+QKDYZs2y&Xv% z0y{oBQSr1%nawnO6t%%R^C!carN##B;u^tkO;tt#^U8^|>GX+e5K-qk#$5{nI~eR0qr#t{{T=YqU_lO)0AtloG<IG;EfDreRha8(aMeDqmtq*_L{7pmy|D7y23h+C&)-CgC-!eoQRouGam3~W zuU}w#oFb<2X~wAmrwZKGKfqIFb2n6W1GuD&4_)9e{qF`xj?B+wt33>mz$<6dg0 zyi2=LO%KgfO@oJm1v-iaty^6pY@`|*&DRB5C>L?^VF{kWGdTwU6fS=KdLW ze>k98@tb0AJ4+&@Q-ez5F(qJ-~Q#W})mb zxPmeOf0~zbjb1^7Z6SAUTHOqD>QATbzB9m@QxSt2QGajPn;xww5A?^-B2ll98b>wIb|$lr|(bJGPt2w z0Lt!jadi|IhkY7AoweJdnkt$C7e?tYwNL=>1`o}9)C(m8X9zcI@(x$`lFD~;Byj3? znJjdmlmXCYzlUb#XMy$d)zYIPx#+J&T^E1l_JAO+%uBF`hJAw3@=xs|H%IuBpvP zLVX`S4T2w|sLcpwfbzDa<;`N(_@$UVjZ8sUq2sL?v{M_jPU?Yzhi%&gREtsBJ zB2jDi09tTyNu8J4ji#b_=Q2jqpbpE$^IsqpnNM=uDFtdOdI(T`6p3t{N%p89b@ z-6{+pBF%pKFzu0Cm!;O0Q&eBZjSsSewRZL4EiY9b0lA&T6NmlumK4(mJ|)k~pf9^$sJ42Q3YE(hK% z5m`DEJldW@y@ChcfmC0M2?=t?CL0qe!=!q{bAzZ!U~kdHSa?nFlaaOADmoT-JuV@vh2cW?e4*B(B5Dt#q! z_WgkW)V@5}-e24A-2CuYUC^+}UmxH=+7=I3R!w<|%`B}_t!ASIz-1(N?P{Nv+c{-) zX6~JNlV_CcQdLY6r$wcm;S@RPs0h+#XSUtkG3`Iq3wf#n2!;o2L|&xW%Z+CF_x^bk z;#9)cl+|o~PHAHvcnTxMyyt;{J){SNK?YA12@| zfY42)o(t-L5 zMW6)_`m0hj9~&3m?q08(v-gO>!eiB!$z|97Ybop?uBxQ)mfCT#4Fx8PW+hbh^?B*4 zOGr;o!gt$UvW~SXRnv2juJSl~ZVsQ!eHt^darNQwG%X$4e9o_6LvIkVJ0adxC(GOJ zg9~$1qLhgfvV!KTA~#D8(b;m1k}^a0;tMv5ksp-bkjH_V;#{r9GlV}R`GF+mKpb7I zkfuV|M4B`KE(^mbx?@y_%9w$F+?V>Vhnjcgwn39ay5`SQtnMlocJB((q|8DOK&m-M z8{7!7gixBD9A_6B`FQE#x^523By?^fKo<0@<`;cCa!#Ey_)gWTl25*-knV?3eXgzA zd@l$e+l>~6n)!$DJNNh%aAd_76~TZ$6##dSZ6T7{xK$M)Y+#CB4tF{;(fIC}v)8YP z76I(WDrb77ERFiaD*CG}(J&~w?9tJDV1>=Tcl(0_xM{v~J+%$_qy;zM$HNdqN>R_QW+pZbHd@cUpC`#} z`HO-(sv6OV*WC45)!U?Lj>5OIT|{4I%Fmw8P0(~fGL{OD&)B;t5rrQFAh8ky^ h$F}O<13Ij(FR{B|i2zJgtUWoCBNIIcSMkB={|8js>s$Z; literal 0 HcmV?d00001 From 3824ebe2305afa1b014f39448aaf7bb2720f2367 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 20 Jan 2026 16:09:38 +0000 Subject: [PATCH 960/962] chore(release): version 21.15.0 [skip ci] ## [21.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.14.0...v21.15.0) (2026-01-20) ### Features * Added link to AWS EKS workshops ([#3631](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3631)) ([c2d3b48](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c2d3b488ce9c63fabeaaded3f369b26a0902126a)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f316c9de9..10fecedac1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.14.0...v21.15.0) (2026-01-20) + +### Features + +* Added link to AWS EKS workshops ([#3631](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3631)) ([c2d3b48](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/c2d3b488ce9c63fabeaaded3f369b26a0902126a)) + ## [21.14.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.13.0...v21.14.0) (2026-01-13) ### Features From 8a833809b9314a57d93b08597679fd4b2ea2af65 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 20 Jan 2026 10:44:41 -0600 Subject: [PATCH 961/962] fix: Move EKS workshop banner up the README (#3633) --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 74f71cd212..e45cf55a33 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,11 @@ Terraform module which creates Amazon EKS (Kubernetes) resources - [Upgrade to v20.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-20.0.md) - [Upgrade to v21.x](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/UPGRADE-21.0.md) + +

    + Register for an AWS hosted workshop: Click to view the full schedule of free workshops, from core concepts to advanced use cases +

    + ### External Documentation Please note that we strive to provide a comprehensive suite of documentation for __*configuring and utilizing the module(s)*__ defined here, and that documentation regarding EKS (including EKS managed node group, self managed node group, and Fargate profile) and/or Kubernetes features, usage, etc. are better left up to their respective sources: @@ -24,9 +29,6 @@ Please note that we strive to provide a comprehensive suite of documentation for - [AWS EKS Documentation](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Kubernetes Documentation](https://kubernetes.io/docs/home/) - -Register for an AWS hosted workshop: Click to view the full schedule of free workshops, from core concepts to advanced use cases - ## Usage ### EKS Auto Mode From 42693d40bceb3ad80d49b0574cc3046455c2def6 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 20 Jan 2026 16:46:12 +0000 Subject: [PATCH 962/962] chore(release): version 21.15.1 [skip ci] ## [21.15.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.15.0...v21.15.1) (2026-01-20) ### Bug Fixes * Move EKS workshop banner up the README ([#3633](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3633)) ([8a83380](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8a833809b9314a57d93b08597679fd4b2ea2af65)) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10fecedac1..5583c68217 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [21.15.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.15.0...v21.15.1) (2026-01-20) + +### Bug Fixes + +* Move EKS workshop banner up the README ([#3633](https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3633)) ([8a83380](https://github.com/terraform-aws-modules/terraform-aws-eks/commit/8a833809b9314a57d93b08597679fd4b2ea2af65)) + ## [21.15.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v21.14.0...v21.15.0) (2026-01-20) ### Features